Skip to content

Commit 830b7d3

Browse files
TobiGrTobiGr
authored
Merge pull request #267 from mauriciocolli/hotfixes
Hotfixes for YouTube and improve unavailable cases
2 parents fcbc96a + 5686a6f commit 830b7d3

14 files changed

Lines changed: 298 additions & 101 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,16 @@ public class Response {
1515
private final Map<String, List<String>> responseHeaders;
1616
private final String responseBody;
1717

18-
public Response(int responseCode, String responseMessage, Map<String, List<String>> responseHeaders, @Nullable String responseBody) {
18+
private final String latestUrl;
19+
20+
public Response(int responseCode, String responseMessage, Map<String, List<String>> responseHeaders,
21+
@Nullable String responseBody, @Nullable String latestUrl) {
1922
this.responseCode = responseCode;
2023
this.responseMessage = responseMessage;
2124
this.responseHeaders = responseHeaders != null ? responseHeaders : Collections.<String, List<String>>emptyMap();
2225

2326
this.responseBody = responseBody == null ? "" : responseBody;
27+
this.latestUrl = latestUrl;
2428
}
2529

2630
public int responseCode() {
@@ -40,6 +44,16 @@ public String responseBody() {
4044
return responseBody;
4145
}
4246

47+
/**
48+
* Used for detecting a possible redirection, limited to the latest one.
49+
*
50+
* @return latest url known right before this response object was created
51+
*/
52+
@Nonnull
53+
public String latestUrl() {
54+
return latestUrl;
55+
}
56+
4357
/*//////////////////////////////////////////////////////////////////////////
4458
// Utils
4559
//////////////////////////////////////////////////////////////////////////*/
@@ -54,7 +68,8 @@ public String responseBody() {
5468
@Nullable
5569
public String getHeader(String name) {
5670
for (Map.Entry<String, List<String>> headerEntry : responseHeaders.entrySet()) {
57-
if (headerEntry.getKey().equalsIgnoreCase(name)) {
71+
final String key = headerEntry.getKey();
72+
if (key != null && key.equalsIgnoreCase(name)) {
5873
if (headerEntry.getValue().size() > 0) {
5974
return headerEntry.getValue().get(0);
6075
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
5858
final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
5959

6060
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
61-
6261
initialData = ajaxJson.getObject(1).getObject("response");
62+
YoutubeParsingHelper.defaultAlertsCheck(initialData);
6363
}
6464

6565

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
1111
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
1212
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
13+
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
1314
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1415
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
1516
import org.schabi.newpipe.extractor.utils.Utils;
@@ -39,6 +40,8 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
3940
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
4041

4142
initialData = ajaxJson.getObject(1).getObject("response");
43+
YoutubeParsingHelper.defaultAlertsCheck(initialData);
44+
4245
playlistInfo = getPlaylistInfo();
4346
}
4447

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ public String getUrl() throws ParsingException {
6363
}
6464

6565
@Override
66-
public String getSearchSuggestion() {
66+
public String getSearchSuggestion() throws ParsingException {
6767
JsonObject showingResultsForRenderer = initialData.getObject("contents")
6868
.getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
6969
.getObject("sectionListRenderer").getArray("contents").getObject(0)
@@ -114,7 +114,7 @@ public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, Extra
114114
return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations")));
115115
}
116116

117-
private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException {
117+
private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException {
118118
collector.reset();
119119

120120
final TimeAgoParser timeAgoParser = getTimeAgoParser();

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.schabi.newpipe.extractor.NewPipe;
1212
import org.schabi.newpipe.extractor.StreamingService;
1313
import org.schabi.newpipe.extractor.downloader.Downloader;
14+
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
1415
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
1516
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1617
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@@ -32,6 +33,7 @@
3233
import org.schabi.newpipe.extractor.stream.StreamType;
3334
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
3435
import org.schabi.newpipe.extractor.stream.VideoStream;
36+
import org.schabi.newpipe.extractor.utils.JsonUtils;
3537
import org.schabi.newpipe.extractor.utils.Parser;
3638
import org.schabi.newpipe.extractor.utils.Utils;
3739

@@ -564,8 +566,12 @@ public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException {
564566
*/
565567
@Override
566568
public String getErrorMessage() {
567-
return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus")
568-
.getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason"));
569+
try {
570+
return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus")
571+
.getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason"));
572+
} catch (ParsingException e) {
573+
return null;
574+
}
569575
}
570576

571577
/*//////////////////////////////////////////////////////////////////////////
@@ -615,6 +621,13 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
615621

616622
playerResponse = getPlayerResponse();
617623

624+
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.DEFAULT_EMPTY);
625+
final String status = playabilityStatus.getString("status");
626+
if (status != null && status.toLowerCase().equals("error")) {
627+
final String reason = playabilityStatus.getString("reason");
628+
throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
629+
}
630+
618631
if (decryptionCode.isEmpty()) {
619632
decryptionCode = loadDecryptionCode(playerUrl);
620633
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java‎

Lines changed: 60 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5-
65
import org.schabi.newpipe.extractor.exceptions.ParsingException;
76
import org.schabi.newpipe.extractor.localization.DateWrapper;
87
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
@@ -13,10 +12,11 @@
1312
import org.schabi.newpipe.extractor.utils.Utils;
1413

1514
import javax.annotation.Nullable;
15+
import java.text.SimpleDateFormat;
16+
import java.util.Calendar;
17+
import java.util.Date;
1618

17-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
18-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
19-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
19+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*;
2020

2121
/*
2222
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@@ -86,7 +86,9 @@ public String getName() throws ParsingException {
8686

8787
@Override
8888
public long getDuration() throws ParsingException {
89-
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
89+
if (getStreamType() == StreamType.LIVE_STREAM || isPremiere()) {
90+
return -1;
91+
}
9092

9193
String duration = null;
9294

@@ -165,7 +167,16 @@ public String getUploaderUrl() throws ParsingException {
165167

166168
@Nullable
167169
@Override
168-
public String getTextualUploadDate() {
170+
public String getTextualUploadDate() throws ParsingException {
171+
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
172+
return null;
173+
}
174+
175+
if (isPremiere()) {
176+
final Date date = getDateFromPremiere().getTime();
177+
return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date);
178+
}
179+
169180
try {
170181
return getTextFromObject(videoInfo.getObject("publishedTimeText"));
171182
} catch (Exception e) {
@@ -177,7 +188,15 @@ public String getTextualUploadDate() {
177188
@Nullable
178189
@Override
179190
public DateWrapper getUploadDate() throws ParsingException {
180-
String textualUploadDate = getTextualUploadDate();
191+
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
192+
return null;
193+
}
194+
195+
if (isPremiere()) {
196+
return new DateWrapper(getDateFromPremiere());
197+
}
198+
199+
final String textualUploadDate = getTextualUploadDate();
181200
if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) {
182201
try {
183202
return timeAgoParser.parse(textualUploadDate);
@@ -194,11 +213,22 @@ public long getViewCount() throws ParsingException {
194213
if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) {
195214
return -1;
196215
}
197-
String viewCount = getTextFromObject(videoInfo.getObject("viewCountText"));
216+
217+
final JsonObject viewCountObject = videoInfo.getObject("viewCountText");
218+
if (viewCountObject == null) {
219+
// This object is null when a video has its views hidden.
220+
return -1;
221+
}
222+
223+
final String viewCount = getTextFromObject(viewCountObject);
224+
225+
if (viewCount.toLowerCase().contains("no views")) {
226+
return 0;
227+
} else if (viewCount.toLowerCase().contains("recommended")) {
228+
return -1;
229+
}
198230

199231
return Long.parseLong(Utils.removeNonDigitCharacters(viewCount));
200-
} catch (NumberFormatException e) {
201-
return -1;
202232
} catch (Exception e) {
203233
throw new ParsingException("Could not get view count", e);
204234
}
@@ -225,7 +255,26 @@ private boolean isPremium() {
225255
return true;
226256
}
227257
}
228-
} catch (Exception ignored) {}
258+
} catch (Exception ignored) {
259+
}
229260
return false;
230261
}
262+
263+
private boolean isPremiere() {
264+
return videoInfo.has("upcomingEventData");
265+
}
266+
267+
private Calendar getDateFromPremiere() throws ParsingException {
268+
final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData");
269+
final String startTime = upcomingEventData.getString("startTime");
270+
271+
try {
272+
final long startTimeTimestamp = Long.parseLong(startTime);
273+
final Calendar calendar = Calendar.getInstance();
274+
calendar.setTime(new Date(startTimeTimestamp * 1000L));
275+
return calendar;
276+
} catch (Exception e) {
277+
throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\"");
278+
}
279+
}
231280
}

0 commit comments

Comments
 (0)