Skip to content

Commit 2037039

Browse files
Theta-DevTheta-Dev
committed
fix: add support for CarouselHeaderRenderer
1 parent 7dba6e3 commit 2037039

6 files changed

Lines changed: 1274 additions & 53 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 96 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
44
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
5-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
65
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
76
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
87
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
@@ -34,6 +33,7 @@
3433
import java.util.ArrayList;
3534
import java.util.List;
3635
import java.util.Objects;
36+
import java.util.Optional;
3737

3838
import javax.annotation.Nonnull;
3939
import javax.annotation.Nullable;
@@ -60,6 +60,8 @@
6060

6161
public class YoutubeChannelExtractor extends ChannelExtractor {
6262
private JsonObject initialData;
63+
private Optional<JsonObject> channelHeader;
64+
private boolean isCarouselHeader = false;
6365
private JsonObject videoTab;
6466

6567
/**
@@ -189,6 +191,30 @@ private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonRespons
189191
}
190192
}
191193

194+
@Nonnull
195+
private Optional<JsonObject> getChannelHeader() {
196+
if (channelHeader == null) {
197+
final JsonObject h = initialData.getObject("header");
198+
199+
if (h.has("c4TabbedHeaderRenderer")) {
200+
channelHeader = Optional.of(h.getObject("c4TabbedHeaderRenderer"));
201+
} else if (h.has("carouselHeaderRenderer")) {
202+
isCarouselHeader = true;
203+
channelHeader = h.getObject("carouselHeaderRenderer")
204+
.getArray("contents")
205+
.stream()
206+
.filter(JsonObject.class::isInstance)
207+
.map(JsonObject.class::cast)
208+
.filter(itm -> itm.has("topicChannelDetailsRenderer"))
209+
.findFirst()
210+
.map(itm -> itm.getObject("topicChannelDetailsRenderer"));
211+
} else {
212+
channelHeader = Optional.empty();
213+
}
214+
}
215+
return channelHeader;
216+
}
217+
192218
@Nonnull
193219
@Override
194220
public String getUrl() throws ParsingException {
@@ -202,58 +228,61 @@ public String getUrl() throws ParsingException {
202228
@Nonnull
203229
@Override
204230
public String getId() throws ParsingException {
205-
final String channelId = initialData.getObject("header")
206-
.getObject("c4TabbedHeaderRenderer")
207-
.getString("channelId", "");
208-
209-
if (!channelId.isEmpty()) {
210-
return channelId;
211-
} else if (!isNullOrEmpty(redirectedChannelId)) {
212-
return redirectedChannelId;
213-
} else {
214-
throw new ParsingException("Could not get channel id");
215-
}
231+
return getChannelHeader()
232+
.flatMap(header -> Optional.ofNullable(header.getString("channelId")).or(
233+
() -> Optional.ofNullable(header.getObject("navigationEndpoint")
234+
.getObject("browseEndpoint")
235+
.getString("browseId"))
236+
))
237+
.or(() -> Optional.ofNullable(redirectedChannelId))
238+
.orElseThrow(() -> new ParsingException("Could not get channel id"));
216239
}
217240

218241
@Nonnull
219242
@Override
220243
public String getName() throws ParsingException {
221-
try {
222-
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
223-
.getString("title");
224-
} catch (final Exception e) {
225-
throw new ParsingException("Could not get channel name", e);
244+
final String mdName = initialData.getObject("metadata")
245+
.getObject("channelMetadataRenderer")
246+
.getString("title");
247+
if (!isNullOrEmpty(mdName)) {
248+
return mdName;
249+
}
250+
251+
final Optional<JsonObject> header = getChannelHeader();
252+
if (header.isPresent()) {
253+
final Object title = header.get().get("title");
254+
if (title instanceof String) {
255+
return (String) title;
256+
} else if (title instanceof JsonObject) {
257+
final String headerName = getTextFromObject((JsonObject) title);
258+
if (!isNullOrEmpty(headerName)) {
259+
return headerName;
260+
}
261+
}
226262
}
263+
264+
throw new ParsingException("Could not get channel name");
227265
}
228266

229267
@Override
230268
public String getAvatarUrl() throws ParsingException {
231-
try {
232-
final String url = initialData.getObject("header")
233-
.getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails")
234-
.getObject(0).getString("url");
235-
236-
return fixThumbnailUrl(url);
237-
} catch (final Exception e) {
238-
throw new ParsingException("Could not get avatar", e);
239-
}
269+
return getChannelHeader().flatMap(header -> Optional.ofNullable(
270+
header.getObject("avatar").getArray("thumbnails")
271+
.getObject(0).getString("url")
272+
))
273+
.map(YoutubeParsingHelper::fixThumbnailUrl)
274+
.orElseThrow(() -> new ParsingException("Could not get avatar"));
240275
}
241276

242277
@Override
243278
public String getBannerUrl() throws ParsingException {
244-
try {
245-
final String url = initialData.getObject("header")
246-
.getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails")
247-
.getObject(0).getString("url");
248-
249-
if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) {
250-
return null;
251-
}
252-
253-
return fixThumbnailUrl(url);
254-
} catch (final Exception e) {
255-
throw new ParsingException("Could not get banner", e);
256-
}
279+
return getChannelHeader().flatMap(header -> Optional.ofNullable(
280+
header.getObject("banner").getArray("thumbnails")
281+
.getObject(0).getString("url")
282+
))
283+
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
284+
.map(YoutubeParsingHelper::fixThumbnailUrl)
285+
.orElseThrow(() -> new ParsingException("Could not get banner"));
257286
}
258287

259288
@Override
@@ -267,17 +296,25 @@ public String getFeedUrl() throws ParsingException {
267296

268297
@Override
269298
public long getSubscriberCount() throws ParsingException {
270-
final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header")
271-
.getObject("c4TabbedHeaderRenderer");
272-
if (!c4TabbedHeaderRenderer.has("subscriberCountText")) {
273-
return UNKNOWN_SUBSCRIBER_COUNT;
274-
}
275-
try {
276-
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer
277-
.getObject("subscriberCountText")));
278-
} catch (final NumberFormatException e) {
279-
throw new ParsingException("Could not get subscriber count", e);
299+
final Optional<JsonObject> header = getChannelHeader();
300+
if (header.isPresent()) {
301+
JsonObject textObject = null;
302+
303+
if (header.get().has("subscriberCountText")) {
304+
textObject = header.get().getObject("subscriberCountText");
305+
} else if (header.get().has("subtitle")) {
306+
textObject = header.get().getObject("subtitle");
307+
}
308+
309+
if (textObject != null) {
310+
try {
311+
return Utils.mixedNumberWordToLong(getTextFromObject(textObject));
312+
} catch (final NumberFormatException e) {
313+
throw new ParsingException("Could not get subscriber count", e);
314+
}
315+
}
280316
}
317+
return UNKNOWN_SUBSCRIBER_COUNT;
281318
}
282319

283320
@Override
@@ -307,11 +344,17 @@ public String getParentChannelAvatarUrl() {
307344

308345
@Override
309346
public boolean isVerified() throws ParsingException {
310-
final JsonArray badges = initialData.getObject("header")
311-
.getObject("c4TabbedHeaderRenderer")
312-
.getArray("badges");
347+
// The CarouselHeaderRenderer does not contain any verification badges.
348+
// Since it is only shown on YT-internal channels or on channels of large organizations
349+
// broadcasting live events, we can assume the channel to be verified.
350+
if (isCarouselHeader) {
351+
return true;
352+
}
313353

314-
return YoutubeParsingHelper.isVerified(badges);
354+
return getChannelHeader()
355+
.map(header -> header.getArray("badges"))
356+
.map(YoutubeParsingHelper::isVerified)
357+
.orElse(false);
315358
}
316359

317360
@Nonnull

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,4 +648,94 @@ public void testVerified() throws Exception {
648648
assertFalse(extractor.isVerified());
649649
}
650650
}
651+
652+
public static class Coachella implements BaseChannelExtractorTest {
653+
private static YoutubeChannelExtractor extractor;
654+
655+
@BeforeAll
656+
public static void setUp() throws Exception {
657+
YoutubeTestsUtils.ensureStateless();
658+
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "coachella"));
659+
extractor = (YoutubeChannelExtractor) YouTube
660+
.getChannelExtractor("https://www.youtube.com/channel/UCHF66aWLOxBW4l6VkSrS3cQ");
661+
extractor.fetchPage();
662+
}
663+
664+
/*//////////////////////////////////////////////////////////////////////////
665+
// Extractor
666+
//////////////////////////////////////////////////////////////////////////*/
667+
668+
@Test
669+
public void testServiceId() {
670+
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
671+
}
672+
673+
@Test
674+
public void testName() throws Exception {
675+
assertEquals(extractor.getName(), "Coachella");
676+
}
677+
678+
@Test
679+
public void testId() throws Exception {
680+
assertEquals("UCHF66aWLOxBW4l6VkSrS3cQ", extractor.getId());
681+
}
682+
683+
@Test
684+
public void testUrl() throws ParsingException {
685+
assertEquals("https://www.youtube.com/channel/UCHF66aWLOxBW4l6VkSrS3cQ", extractor.getUrl());
686+
}
687+
688+
@Test
689+
public void testOriginalUrl() throws ParsingException {
690+
assertEquals("https://www.youtube.com/channel/UCHF66aWLOxBW4l6VkSrS3cQ", extractor.getOriginalUrl());
691+
}
692+
693+
/*//////////////////////////////////////////////////////////////////////////
694+
// ListExtractor
695+
//////////////////////////////////////////////////////////////////////////*/
696+
697+
@Test
698+
public void testRelatedItems() throws Exception {
699+
defaultTestRelatedItems(extractor);
700+
}
701+
702+
@Test
703+
public void testMoreRelatedItems() throws Exception {
704+
defaultTestMoreItems(extractor);
705+
}
706+
707+
/*//////////////////////////////////////////////////////////////////////////
708+
// ChannelExtractor
709+
//////////////////////////////////////////////////////////////////////////*/
710+
@Override
711+
public void testDescription() {
712+
}
713+
714+
@Test
715+
public void testAvatarUrl() throws Exception {
716+
String avatarUrl = extractor.getAvatarUrl();
717+
assertIsSecureUrl(avatarUrl);
718+
ExtractorAsserts.assertContains("yt3", avatarUrl);
719+
}
720+
721+
@Test
722+
public void testBannerUrl() throws Exception {
723+
// CarouselHeaderRender does not contain a banner
724+
}
725+
726+
@Test
727+
public void testFeedUrl() throws Exception {
728+
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCHF66aWLOxBW4l6VkSrS3cQ", extractor.getFeedUrl());
729+
}
730+
731+
@Test
732+
public void testSubscriberCount() throws Exception {
733+
ExtractorAsserts.assertGreaterOrEqual(2_900_000, extractor.getSubscriberCount());
734+
}
735+
736+
@Test
737+
public void testVerified() throws Exception {
738+
assertTrue(extractor.isVerified());
739+
}
740+
}
651741
}

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/coachella/generated_mock_0.json‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
{
2+
"request": {
3+
"httpMethod": "GET",
4+
"url": "https://www.youtube.com/sw.js",
5+
"headers": {
6+
"Origin": [
7+
"https://www.youtube.com"
8+
],
9+
"Referer": [
10+
"https://www.youtube.com"
11+
],
12+
"Accept-Language": [
13+
"en-GB, en;q\u003d0.9"
14+
]
15+
},
16+
"localization": {
17+
"languageCode": "en",
18+
"countryCode": "GB"
19+
}
20+
},
21+
"response": {
22+
"responseCode": 200,
23+
"responseMessage": "",
24+
"responseHeaders": {
25+
"access-control-allow-credentials": [
26+
"true"
27+
],
28+
"access-control-allow-origin": [
29+
"https://www.youtube.com"
30+
],
31+
"alt-svc": [
32+
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
33+
],
34+
"cache-control": [
35+
"private, max-age\u003d0"
36+
],
37+
"content-type": [
38+
"text/javascript; charset\u003dutf-8"
39+
],
40+
"cross-origin-opener-policy": [
41+
"same-origin; report-to\u003d\"youtube_main\""
42+
],
43+
"date": [
44+
"Sun, 16 Apr 2023 15:33:19 GMT"
45+
],
46+
"expires": [
47+
"Sun, 16 Apr 2023 15:33:19 GMT"
48+
],
49+
"origin-trial": [
50+
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
51+
],
52+
"p3p": [
53+
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
54+
],
55+
"permissions-policy": [
56+
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
57+
],
58+
"report-to": [
59+
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
60+
],
61+
"server": [
62+
"ESF"
63+
],
64+
"set-cookie": [
65+
"YSC\u003dOCGx8FJdx2E; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
66+
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Jul-2020 15:33:19 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
67+
"CONSENT\u003dPENDING+955; expires\u003dTue, 15-Apr-2025 15:33:19 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
68+
],
69+
"strict-transport-security": [
70+
"max-age\u003d31536000"
71+
],
72+
"x-content-type-options": [
73+
"nosniff"
74+
],
75+
"x-frame-options": [
76+
"SAMEORIGIN"
77+
],
78+
"x-xss-protection": [
79+
"0"
80+
]
81+
},
82+
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
83+
"latestUrl": "https://www.youtube.com/sw.js"
84+
}
85+
}

0 commit comments

Comments
 (0)