Skip to content

Commit 6fd9b38

Browse files
TobiGrTobiGr
authored
Merge pull request #287 from mauriciocolli/fix-channel-redirect
[YouTube] Fix channel with redirects directly in the response
2 parents 2cf8cbf + b086e9d commit 6fd9b38

4 files changed

Lines changed: 164 additions & 14 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 65 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5-
65
import org.schabi.newpipe.extractor.StreamingService;
76
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
87
import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -16,13 +15,11 @@
1615
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
1716
import org.schabi.newpipe.extractor.utils.Utils;
1817

19-
import java.io.IOException;
20-
2118
import javax.annotation.Nonnull;
19+
import java.io.IOException;
2220

23-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
24-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
25-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
21+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*;
22+
import static org.schabi.newpipe.extractor.utils.JsonUtils.*;
2623

2724
/*
2825
* Created by Christian Schabesberger on 25.07.16.
@@ -49,15 +46,64 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
4946
private JsonObject initialData;
5047
private JsonObject videoTab;
5148

49+
/**
50+
* Some channels have response redirects and the only way to reliably get the id is by saving it.
51+
*<p>
52+
* "Movies & Shows":
53+
* <pre>
54+
* UCuJcl0Ju-gPDoksRjK1ya-w ┐
55+
* UChBfWrfBXL9wS6tQtgjt_OQ ├ UClgRkhTL3_hImCAmdLfDE4g
56+
* UCok7UTQQEP1Rsctxiv3gwSQ ┘
57+
* </pre>
58+
*/
59+
private String redirectedChannelId;
60+
5261
public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
5362
super(service, linkHandler);
5463
}
5564

5665
@Override
5766
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
58-
final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
67+
String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
68+
JsonArray ajaxJson = null;
69+
70+
int level = 0;
71+
while (level < 3) {
72+
final JsonArray jsonResponse = getJsonResponse(url, getExtractorLocalization());
73+
74+
final JsonObject endpoint = jsonResponse.getObject(1, EMPTY_OBJECT)
75+
.getObject("response", EMPTY_OBJECT).getArray("onResponseReceivedActions", EMPTY_ARRAY)
76+
.getObject(0, EMPTY_OBJECT).getObject("navigateAction", EMPTY_OBJECT)
77+
.getObject("endpoint", EMPTY_OBJECT);
78+
79+
final String webPageType = endpoint
80+
.getObject("commandMetadata", EMPTY_OBJECT)
81+
.getObject("webCommandMetadata", EMPTY_OBJECT)
82+
.getString("webPageType", EMPTY_STRING);
83+
84+
final String browseId = endpoint
85+
.getObject("browseEndpoint", EMPTY_OBJECT)
86+
.getString("browseId", EMPTY_STRING);
87+
88+
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) {
89+
90+
if (!browseId.startsWith("UC")) {
91+
throw new ExtractionException("Redirected id is not pointing to a channel");
92+
}
93+
94+
url = "https://www.youtube.com/channel/" + browseId + "/videos?pbj=1&view=0&flow=grid";
95+
redirectedChannelId = browseId;
96+
level++;
97+
} else {
98+
ajaxJson = jsonResponse;
99+
break;
100+
}
101+
}
102+
103+
if (ajaxJson == null) {
104+
throw new ExtractionException("Could not fetch initial JSON data");
105+
}
59106

60-
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
61107
initialData = ajaxJson.getObject(1).getObject("response");
62108
YoutubeParsingHelper.defaultAlertsCheck(initialData);
63109
}
@@ -84,10 +130,17 @@ public String getUrl() throws ParsingException {
84130
@Nonnull
85131
@Override
86132
public String getId() throws ParsingException {
87-
try {
88-
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("channelId");
89-
} catch (Exception e) {
90-
throw new ParsingException("Could not get channel id", e);
133+
final String channelId = initialData
134+
.getObject("header", EMPTY_OBJECT)
135+
.getObject("c4TabbedHeaderRenderer", EMPTY_OBJECT)
136+
.getString("channelId", EMPTY_STRING);
137+
138+
if (!channelId.isEmpty()) {
139+
return channelId;
140+
} else if (redirectedChannelId != null && !redirectedChannelId.isEmpty()) {
141+
return redirectedChannelId;
142+
} else {
143+
throw new ParsingException("Could not get channel id");
91144
}
92145
}
93146

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -629,7 +629,7 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
629629

630630
playerResponse = getPlayerResponse();
631631

632-
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.DEFAULT_EMPTY);
632+
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.EMPTY_OBJECT);
633633
final String status = playabilityStatus.getString("status");
634634
// If status exist, and is not "OK", throw a ContentNotAvailableException with the reason.
635635
if (status != null && !status.toLowerCase().equals("ok")) {

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111
import java.util.List;
1212

1313
public class JsonUtils {
14-
public static final JsonObject DEFAULT_EMPTY = new JsonObject();
14+
public static final JsonObject EMPTY_OBJECT = new JsonObject();
15+
public static final JsonArray EMPTY_ARRAY = new JsonArray();
16+
public static final String EMPTY_STRING = "";
1517

1618
private JsonUtils() {
1719
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,16 @@
55
import org.schabi.newpipe.DownloaderTestImpl;
66
import org.schabi.newpipe.extractor.NewPipe;
77
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
8+
import org.schabi.newpipe.extractor.channel.ChannelInfo;
89
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
910
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1011
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
1112
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
1213

14+
import java.util.List;
15+
1316
import static org.junit.Assert.*;
17+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
1418
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
1519
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
1620
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
@@ -505,6 +509,97 @@ public void testSubscriberCount() throws Exception {
505509
}
506510
}
507511

512+
/**
513+
* Some VEVO channels will redirect to a new page with a new channel id.
514+
* <p>
515+
* Though, it isn't a simple redirect, but a redirect instruction embed in the response itself, this
516+
* test assure that we account for that.
517+
*/
518+
public static class RedirectedChannel implements BaseChannelExtractorTest {
519+
private static YoutubeChannelExtractor extractor;
520+
521+
@BeforeClass
522+
public static void setUp() throws Exception {
523+
NewPipe.init(DownloaderTestImpl.getInstance());
524+
extractor = (YoutubeChannelExtractor) YouTube
525+
.getChannelExtractor("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ");
526+
extractor.fetchPage();
527+
}
528+
529+
/*//////////////////////////////////////////////////////////////////////////
530+
// Extractor
531+
//////////////////////////////////////////////////////////////////////////*/
532+
533+
@Test
534+
public void testServiceId() {
535+
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
536+
}
537+
538+
@Test
539+
public void testName() throws Exception {
540+
assertEquals("LordiVEVO", extractor.getName());
541+
}
542+
543+
@Test
544+
public void testId() throws Exception {
545+
assertEquals("UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getId());
546+
}
547+
548+
@Test
549+
public void testUrl() throws ParsingException {
550+
assertEquals("https://www.youtube.com/channel/UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getUrl());
551+
}
552+
553+
@Test
554+
public void testOriginalUrl() throws ParsingException {
555+
assertEquals("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ", extractor.getOriginalUrl());
556+
}
557+
558+
/*//////////////////////////////////////////////////////////////////////////
559+
// ListExtractor
560+
//////////////////////////////////////////////////////////////////////////*/
561+
562+
@Test
563+
public void testRelatedItems() throws Exception {
564+
defaultTestRelatedItems(extractor);
565+
}
566+
567+
@Test
568+
public void testMoreRelatedItems() throws Exception {
569+
assertNoMoreItems(extractor);
570+
}
571+
572+
/*//////////////////////////////////////////////////////////////////////////
573+
// ChannelExtractor
574+
//////////////////////////////////////////////////////////////////////////*/
575+
576+
@Test
577+
public void testDescription() throws Exception {
578+
assertEmpty(extractor.getDescription());
579+
}
580+
581+
@Test
582+
public void testAvatarUrl() throws Exception {
583+
String avatarUrl = extractor.getAvatarUrl();
584+
assertIsSecureUrl(avatarUrl);
585+
assertTrue(avatarUrl, avatarUrl.contains("yt3"));
586+
}
587+
588+
@Test
589+
public void testBannerUrl() throws Exception {
590+
assertEmpty(extractor.getBannerUrl());
591+
}
592+
593+
@Test
594+
public void testFeedUrl() throws Exception {
595+
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getFeedUrl());
596+
}
597+
598+
@Test
599+
public void testSubscriberCount() throws Exception {
600+
assertEquals(-1, extractor.getSubscriberCount());
601+
}
602+
}
508603

509604
public static class RandomChannel implements BaseChannelExtractorTest {
510605
private static YoutubeChannelExtractor extractor;

0 commit comments

Comments
 (0)