Skip to content

Commit 00d1ed4

Browse files
mauriciocollimauriciocolli
committed
[YouTube] Fix channel extraction when redirects are in the response
Some redirects were embed directly into the response as instructions for the page, instead of the usual http redirects.
1 parent e7be952 commit 00d1ed4

4 files changed

Lines changed: 140 additions & 10 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5-
65
import org.schabi.newpipe.extractor.StreamingService;
76
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
87
import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -16,13 +15,11 @@
1615
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
1716
import org.schabi.newpipe.extractor.utils.Utils;
1817

19-
import java.io.IOException;
20-
2118
import javax.annotation.Nonnull;
19+
import java.io.IOException;
2220

23-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
24-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
25-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
21+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*;
22+
import static org.schabi.newpipe.extractor.utils.JsonUtils.*;
2623

2724
/*
2825
* Created by Christian Schabesberger on 25.07.16.
@@ -55,9 +52,45 @@ public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHan
5552

5653
@Override
5754
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
58-
final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
55+
String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
56+
JsonArray ajaxJson = null;
57+
58+
int level = 0;
59+
while (level < 3) {
60+
final JsonArray jsonResponse = getJsonResponse(url, getExtractorLocalization());
61+
62+
final JsonObject endpoint = jsonResponse.getObject(1, EMPTY_OBJECT)
63+
.getObject("response", EMPTY_OBJECT).getArray("onResponseReceivedActions", EMPTY_ARRAY)
64+
.getObject(0, EMPTY_OBJECT).getObject("navigateAction", EMPTY_OBJECT)
65+
.getObject("endpoint", EMPTY_OBJECT);
66+
67+
final String webPageType = endpoint
68+
.getObject("commandMetadata", EMPTY_OBJECT)
69+
.getObject("webCommandMetadata", EMPTY_OBJECT)
70+
.getString("webPageType", EMPTY_STRING);
71+
72+
final String browseId = endpoint
73+
.getObject("browseEndpoint", EMPTY_OBJECT)
74+
.getString("browseId", EMPTY_STRING);
75+
76+
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) {
77+
78+
if (!browseId.startsWith("UC")) {
79+
throw new ExtractionException("Redirected id is not pointing to a channel");
80+
}
81+
82+
url = "https://www.youtube.com/channel/" + browseId + "/videos?pbj=1&view=0&flow=grid";
83+
level++;
84+
} else {
85+
ajaxJson = jsonResponse;
86+
break;
87+
}
88+
}
89+
90+
if (ajaxJson == null) {
91+
throw new ExtractionException("Could not fetch initial JSON data");
92+
}
5993

60-
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
6194
initialData = ajaxJson.getObject(1).getObject("response");
6295
YoutubeParsingHelper.defaultAlertsCheck(initialData);
6396
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -621,7 +621,7 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
621621

622622
playerResponse = getPlayerResponse();
623623

624-
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.DEFAULT_EMPTY);
624+
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.EMPTY_OBJECT);
625625
final String status = playabilityStatus.getString("status");
626626
// If status exist, and is not "OK", throw a ContentNotAvailableException with the reason.
627627
if (status != null && !status.toLowerCase().equals("ok")) {

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111
import java.util.List;
1212

1313
public class JsonUtils {
14-
public static final JsonObject DEFAULT_EMPTY = new JsonObject();
14+
public static final JsonObject EMPTY_OBJECT = new JsonObject();
15+
public static final JsonArray EMPTY_ARRAY = new JsonArray();
16+
public static final String EMPTY_STRING = "";
1517

1618
private JsonUtils() {
1719
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,16 @@
55
import org.schabi.newpipe.DownloaderTestImpl;
66
import org.schabi.newpipe.extractor.NewPipe;
77
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
8+
import org.schabi.newpipe.extractor.channel.ChannelInfo;
89
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
910
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1011
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
1112
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
1213

14+
import java.util.List;
15+
1316
import static org.junit.Assert.*;
17+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
1418
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
1519
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
1620
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
@@ -505,6 +509,97 @@ public void testSubscriberCount() throws Exception {
505509
}
506510
}
507511

512+
/**
513+
* Some VEVO channels will redirect to a new page with a new channel id.
514+
* <p>
515+
* Though, it isn't a simple redirect, but a redirect instruction embed in the response itself, this
516+
* test assure that we account for that.
517+
*/
518+
public static class RedirectedChannel implements BaseChannelExtractorTest {
519+
private static YoutubeChannelExtractor extractor;
520+
521+
@BeforeClass
522+
public static void setUp() throws Exception {
523+
NewPipe.init(DownloaderTestImpl.getInstance());
524+
extractor = (YoutubeChannelExtractor) YouTube
525+
.getChannelExtractor("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ");
526+
extractor.fetchPage();
527+
}
528+
529+
/*//////////////////////////////////////////////////////////////////////////
530+
// Extractor
531+
//////////////////////////////////////////////////////////////////////////*/
532+
533+
@Test
534+
public void testServiceId() {
535+
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
536+
}
537+
538+
@Test
539+
public void testName() throws Exception {
540+
assertEquals("LordiVEVO", extractor.getName());
541+
}
542+
543+
@Test
544+
public void testId() throws Exception {
545+
assertEquals("UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getId());
546+
}
547+
548+
@Test
549+
public void testUrl() throws ParsingException {
550+
assertEquals("https://www.youtube.com/channel/UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getUrl());
551+
}
552+
553+
@Test
554+
public void testOriginalUrl() throws ParsingException {
555+
assertEquals("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ", extractor.getOriginalUrl());
556+
}
557+
558+
/*//////////////////////////////////////////////////////////////////////////
559+
// ListExtractor
560+
//////////////////////////////////////////////////////////////////////////*/
561+
562+
@Test
563+
public void testRelatedItems() throws Exception {
564+
defaultTestRelatedItems(extractor);
565+
}
566+
567+
@Test
568+
public void testMoreRelatedItems() throws Exception {
569+
assertNoMoreItems(extractor);
570+
}
571+
572+
/*//////////////////////////////////////////////////////////////////////////
573+
// ChannelExtractor
574+
//////////////////////////////////////////////////////////////////////////*/
575+
576+
@Test
577+
public void testDescription() throws Exception {
578+
assertEmpty(extractor.getDescription());
579+
}
580+
581+
@Test
582+
public void testAvatarUrl() throws Exception {
583+
String avatarUrl = extractor.getAvatarUrl();
584+
assertIsSecureUrl(avatarUrl);
585+
assertTrue(avatarUrl, avatarUrl.contains("yt3"));
586+
}
587+
588+
@Test
589+
public void testBannerUrl() throws Exception {
590+
assertEmpty(extractor.getBannerUrl());
591+
}
592+
593+
@Test
594+
public void testFeedUrl() throws Exception {
595+
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getFeedUrl());
596+
}
597+
598+
@Test
599+
public void testSubscriberCount() throws Exception {
600+
assertEquals(-1, extractor.getSubscriberCount());
601+
}
602+
}
508603

509604
public static class RandomChannel implements BaseChannelExtractorTest {
510605
private static YoutubeChannelExtractor extractor;

0 commit comments

Comments
 (0)