Skip to content

[YouTube] Fix parsing subscribers count of channels with pronouns #1318

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TobiGr merged 2 commits into from
Jul 7, 2025
Merged

[YouTube] Fix parsing subscribers count of channels with pronouns #1318

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
53757ae
[YouTube] Support pronouns in subs count parsing of page channel headers
AudricV Jul 7, 2025
8bab96a
[YouTube] Add a test of a channel with pronouns
AudricV Jul 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 14 additions & 23 deletions ...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,33 +273,24 @@ private long getSubscriberCountFromPageChannelHeader(@Nonnull final JsonObject h
.getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(METADATA);
if (metadataObject.has("contentMetadataViewModel")) {
final JsonArray metadataPart = metadataObject.getObject("contentMetadataViewModel")
.getArray("metadataRows")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(metadataRow -> metadataRow.getArray("metadataParts"))
/*
Find metadata parts which have two elements: channel handle and subscriber
count.

On autogenerated music channels, the subscriber count is not shown with this
header.

Use the first metadata parts object found.
*/
.filter(metadataParts -> metadataParts.size() == 2)
.findFirst()
.orElse(null);
if (metadataPart == null) {
// As the parsing of the metadata parts object needed to get the subscriber count
// is fragile, return UNKNOWN_SUBSCRIBER_COUNT when it cannot be got
final JsonArray metadataRows = metadataObject.getObject("contentMetadataViewModel")
.getArray("metadataRows");

// Extract the latest metadata part, it contains in general two elements: the
// subscribers and videos counts
final JsonArray lastMetadataRowParts = metadataRows.getObject(
Math.max(0, metadataRows.size() - 1))
.getArray("metadataParts");

if (lastMetadataRowParts.size() < 2) {
// For autogenerated channels such as artist ones, subscribers count isn't returned,
// so the metadata part has a lower size
return UNKNOWN_SUBSCRIBER_COUNT;
}

try {
// The subscriber count is at the same position for all languages as of 02/03/2024
return Utils.mixedNumberWordToLong(metadataPart.getObject(0)
// The subscriber count is at the same position for all languages as of 07/07/2025
return Utils.mixedNumberWordToLong(lastMetadataRowParts.getObject(0)
.getObject("text")
.getString(CONTENT));
} catch (final NumberFormatException e) {
Expand Down
95 changes: 95 additions & 0 deletions .../test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1026,4 +1026,99 @@ public void testOriginalUrl() throws Exception {
extractor.getOriginalUrl());
}
}

static class ChannelWithPronouns implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor;

@BeforeAll
static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelWithPronouns"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/@ShempOfficial");
extractor.fetchPage();
}

@Override
@Test
public void testServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}

@Override
@Test
public void testName() throws Exception {
assertEquals("Shemp", extractor.getName());
}

@Override
@Test
public void testId() throws Exception {
assertEquals("UCEAXWzgcuF6XEiJcszhikQA", extractor.getId());
}

@Override
@Test
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCEAXWzgcuF6XEiJcszhikQA", extractor.getUrl());
}

@Override
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/@ShempOfficial", extractor.getOriginalUrl());
}

@Override
@Test
public void testDescription() throws Exception {
assertContains("HEY! HEY YOU! YEAH YOU!", extractor.getDescription());
}

@Override
@Test
public void testAvatars() throws Exception {
YoutubeTestsUtils.testImages(extractor.getAvatars());
}

@Override
@Test
public void testBanners() throws Exception {
YoutubeTestsUtils.testImages(extractor.getBanners());
}

@Override
@Test
public void testFeedUrl() throws Exception {
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCEAXWzgcuF6XEiJcszhikQA", extractor.getFeedUrl());
}

@Override
@Test
public void testSubscriberCount() throws Exception {
ExtractorAsserts.assertGreaterOrEqual(7_000, extractor.getSubscriberCount());
}

@Override
@Test
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}

@Override
@Test
public void testTabs() throws Exception {
assertTabsContain(extractor.getTabs(),
ChannelTabs.VIDEOS, ChannelTabs.LIVESTREAMS, ChannelTabs.PLAYLISTS);
assertTrue(extractor.getTabs().stream()
.filter(it -> ChannelTabs.VIDEOS.equals(it.getContentFilters().get(0)))
.allMatch(ReadyChannelTabListLinkHandler.class::isInstance));
}

@Override
@Test
public void testTags() throws Exception {
assertTrue(extractor.getTags().contains("shemp"));
}
}
}
93 changes: 93 additions & 0 deletions ...pe/extractor/services/youtube/extractor/channel/channelWithPronouns/generated_mock_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Referer": [
"https://www.youtube.com"
],
"Origin": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"access-control-allow-credentials": [
"true"
],
"access-control-allow-origin": [
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
],
"cache-control": [
"private, max-age\u003d0"
],
"content-security-policy": [
"require-trusted-types-for \u0027script\u0027"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Mon, 07 Jul 2025 16:58:02 GMT"
],
"document-policy": [
"include-js-call-stacks-in-crash-reports"
],
"expires": [
"Mon, 07 Jul 2025 16:58:02 GMT"
],
"origin-trial": [
"AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factors\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"reporting-endpoints": [
"default\u003d\"/web-reports?context\u003deJwNzltIk3EcxvH-PhT2vnPv-_sbVCqxTMrMydRmZWqhtQ5YGRSIh7aczsypuba1zO66ESQK6aBZEYRKkR0ES9S6KAhKCTwQmCYoIYJ5uMjyUPa7-Nx9eXiUltWDplKR6i4T9pUyEXW4XLy4WinytnnEwS8XRa3NK-abvOJZp1c09PjEsUGfmDL5RW6kX9wd9IuH3_zCPRoQS-MB8UDPDnKFZQcFmCMiO6gzR4HLpeBXvYK5YQWBGQW5O1T0H1UxdVxFaZ6K9mYVna9U1A2pGJlQ4cs1YNZhgNVnQMW8AZNZIeiuD0FffwhiTxmRUmtES4cRWR-NuMmKl42Y_mtEUqqG1kMa_EUaTBUaVL-GhhoN8Xc0jNdrCH6kYf0nDTVjGrqjdAyk6wh26FAYruiw3tLRd0_HyR4dr1ljr477P3QM_9axd1FHfijBF0Po2klwWAlJyYSqVMK1fYS5DELjacKInRDtJFhchMxiwssSgvM8obqaO9ZVS0i_TpipI6TdJigNhOEmwvdmwrsnhNCnhPBWQlsb4UQ7t28IK28J6mfC416CHCBETBKmpwiYJ4z9IRgWCUYm2Tq2gYWxTczEothWFsNimYUlsCS2m6WwNJbODjAbO7fEf5d54x_hxioJRUhMBEkY10q4N0rMhkl0hEsEmyRsmyW-sv1REt3REgvbJayJEvXs5x6Jf8kSW1IkKtMk8jMkmlmVTWKIjbLYIxK7MiVy2BlWwAqZi7mZh_lZFZOG4IaF5-_X6LNjEx9EpPlyhc_rKyiKu1RUYHZ5Ksq95qLyQrPTU-ItcZ512xMsCVZLUqIlLt5iv2D5D2YQ1as\""
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dFqJ3vRvm-Gk; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dTue, 11-Oct-2022 16:58:02 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}
89 changes: 89 additions & 0 deletions ...pe/extractor/services/youtube/extractor/channel/channelWithPronouns/generated_mock_1.json
View file
Open in desktop

Large diffs are not rendered by default.

Loading
Loading