Skip to content

Commit 31bf704

Browse files
AudricVAudricV
authored
Merge pull request #944 from Theta-Dev/fix-rich-grid-renderer
[YouTube] Support richGridRenderer on channel page to fix extraction of videos in channels
2 parents 4cae66f + e4c24d4 commit 31bf704

30 files changed

Lines changed: 798 additions & 831 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -332,16 +332,20 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
332332
Page nextPage = null;
333333

334334
if (getVideoTab() != null) {
335-
final JsonObject gridRenderer = getVideoTab().getObject("content")
335+
final JsonObject tabContent = getVideoTab().getObject("content");
336+
JsonArray items = tabContent
336337
.getObject("sectionListRenderer")
337338
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
338-
.getArray("contents").getObject(0).getObject("gridRenderer");
339+
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
340+
341+
if (items.isEmpty()) {
342+
items = tabContent.getObject("richGridRenderer").getArray("contents");
343+
}
339344

340345
final List<String> channelIds = new ArrayList<>();
341346
channelIds.add(getName());
342347
channelIds.add(getUrl());
343-
final JsonObject continuation = collectStreamsFrom(collector, gridRenderer
344-
.getArray("items"), channelIds);
348+
final JsonObject continuation = collectStreamsFrom(collector, items, channelIds);
345349

346350
nextPage = getNextPageFrom(continuation, channelIds);
347351
}
@@ -433,6 +437,21 @@ public String getUploaderUrl() {
433437
return uploaderUrl;
434438
}
435439
});
440+
} else if (video.has("richItemRenderer")) {
441+
collector.commit(new YoutubeStreamInfoItemExtractor(
442+
video.getObject("richItemRenderer")
443+
.getObject("content").getObject("videoRenderer"), timeAgoParser) {
444+
@Override
445+
public String getUploaderName() {
446+
return uploaderName;
447+
}
448+
449+
@Override
450+
public String getUploaderUrl() {
451+
return uploaderUrl;
452+
}
453+
});
454+
436455
} else if (video.has("continuationItemRenderer")) {
437456
continuation = video.getObject("continuationItemRenderer");
438457
}

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_0.json‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,23 +37,20 @@
3737
"content-type": [
3838
"text/javascript; charset\u003dutf-8"
3939
],
40-
"critical-ch": [
41-
"Sec-CH-UA-Arch, Sec-CH-UA-Bitness, Sec-CH-UA-Full-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Model, Sec-CH-UA-Platform, Sec-CH-UA-Platform-Version"
42-
],
4340
"cross-origin-opener-policy-report-only": [
4441
"same-origin; report-to\u003d\"youtube_main\""
4542
],
4643
"date": [
47-
"Fri, 12 Aug 2022 17:16:25 GMT"
44+
"Wed, 02 Nov 2022 17:40:55 GMT"
4845
],
4946
"expires": [
50-
"Fri, 12 Aug 2022 17:16:25 GMT"
47+
"Wed, 02 Nov 2022 17:40:55 GMT"
5148
],
5249
"p3p": [
5350
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
5451
],
5552
"permissions-policy": [
56-
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
53+
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
5754
],
5855
"report-to": [
5956
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
@@ -62,9 +59,9 @@
6259
"ESF"
6360
],
6461
"set-cookie": [
65-
"YSC\u003dH65DWcDwcL0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
66-
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 16-Nov-2019 17:16:25 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
67-
"CONSENT\u003dPENDING+212; expires\u003dSun, 11-Aug-2024 17:16:25 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
62+
"YSC\u003d6aCd9lqn2z8; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
63+
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dThu, 06-Feb-2020 17:40:55 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
64+
"CONSENT\u003dPENDING+174; expires\u003dFri, 01-Nov-2024 17:40:54 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
6865
],
6966
"strict-transport-security": [
7067
"max-age\u003d31536000"

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_1.json‎

Lines changed: 6 additions & 6 deletions
Large diffs are not rendered by default.

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_2.json‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
"https://www.youtube.com"
1414
],
1515
"X-YouTube-Client-Version": [
16-
"2.20220809.02.00"
16+
"2.20221101.00.00"
1717
],
1818
"Content-Type": [
1919
"application/json"
@@ -205,13 +205,13 @@
205205
48,
206206
50,
207207
50,
208+
49,
209+
49,
208210
48,
209-
56,
210-
48,
211-
57,
211+
49,
212212
46,
213213
48,
214-
50,
214+
48,
215215
46,
216216
48,
217217
48,
@@ -338,10 +338,10 @@
338338
"application/json; charset\u003dUTF-8"
339339
],
340340
"date": [
341-
"Fri, 12 Aug 2022 17:16:25 GMT"
341+
"Wed, 02 Nov 2022 17:40:55 GMT"
342342
],
343343
"expires": [
344-
"Fri, 12 Aug 2022 17:16:25 GMT"
344+
"Wed, 02 Nov 2022 17:40:55 GMT"
345345
],
346346
"p3p": [
347347
"CP\u003d\"This is not a P3P policy! See g.co/p3phelp for more info.\""
@@ -350,7 +350,7 @@
350350
"scaffolding on HTTPServer2"
351351
],
352352
"set-cookie": [
353-
"CONSENT\u003dPENDING+061; expires\u003dSun, 11-Aug-2024 17:16:25 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
353+
"CONSENT\u003dPENDING+984; expires\u003dFri, 01-Nov-2024 17:40:55 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
354354
],
355355
"vary": [
356356
"Origin",
@@ -367,7 +367,7 @@
367367
"0"
368368
]
369369
},
370-
"responseBody": "{\"responseContext\":{\"visitorData\":\"Cgs0d1FhenR6RXVOayjpldqXBg%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20220809.02.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"ResolveUrl_rid\",\"value\":\"0xc781391fa55a10e4\"}]},{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24139339,24209350,23940248,24187377,1714242,24002025,24164186,24216166,24199724,24199710,24135943,24226335,24220088,24085811,24077241,23998056,24245746,24253963,24235348,24036948,24007246,24225483,39321934,24246427,24222379,24004644,24244808,24241090,24114244,24219713,24166867,24080738,24135310,24238983,24245799,23918597,24260692,23744176,24241936,24199774,24254429,24219033,23983296,24186125,24230625,24140247,24169501,24248231,24227844,24198739,24239147,24077266,24181174,24252017,24034168,23986031,23966208,23885487,24223124,24250883,24211177,23946420,23804281,24214092,24241165,24248385,24161116,24246938,24191629,24001373,24152442,24215196,24187043,24145390,24037231,24229161,24002022,24185614,23882503,24120819,24227532,23934970\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20220805\"},{\"key\":\"client.name\",\"value\":\"WEB\"},{\"key\":\"client.fexp\",\"value\":\"24139339,24209350,23940248,24187377,1714242,24002025,24164186,24216166,24199724,24199710,24135943,24226335,24220088,24085811,24077241,23998056,24245746,24253963,24235348,24036948,24007246,24225483,39321934,24246427,24222379,24004644,24244808,24241090,24114244,24219713,24166867,24080738,24135310,24238983,24245799,23918597,24260692,23744176,24241936,24199774,24254429,24219033,23983296,24186125,24230625,24140247,24169501,24248231,24227844,24198739,24239147,24077266,24181174,24252017,24034168,23986031,23966208,23885487,24223124,24250883,24211177,23946420,23804281,24214092,24241165,24248385,24161116,24246938,24191629,24001373,24152442,24215196,24187043,24145390,24037231,24229161,24002022,24185614,23882503,24120819,24227532,23934970\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"endpoint\":{\"clickTrackingParams\":\"IhMI5ZeWiefB-QIVGF6yCh0ZxgVhMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/youtubei/v1/navigation/resolve_url\",\"webPageType\":\"WEB_PAGE_TYPE_CHANNEL\",\"rootVe\":3611,\"apiUrl\":\"/youtubei/v1/browse\"},\"resolveUrlCommandMetadata\":{\"isVanityUrl\":true}},\"browseEndpoint\":{\"browseId\":\"UC6nSFpj9HTCZ5t-N3Rm3-HA\",\"params\":\"EgC4AQDyBgQKAjIA\"}}}",
370+
"responseBody": "{\"responseContext\":{\"visitorData\":\"CgtJUUxxcnRoM214WSin14qbBg%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20221101.00.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"ResolveUrl_rid\",\"value\":\"0x7b414305caad8bf8\"}]},{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"1714257,23804281,23882502,23885487,23918597,23934970,23946420,23966208,23983296,23986016,23998056,24001373,24002022,24002025,24004644,24007246,24034168,24036947,24077241,24080738,24120820,24135310,24140247,24152443,24161116,24162920,24164186,24166867,24169501,24181174,24184446,24185614,24187043,24187377,24191629,24199724,24206234,24209350,24211178,24219713,24229161,24230618,24241378,24246502,24248092,24254502,24255165,24255543,24255545,24260783,24262346,24263796,24265820,24267564,24267570,24268142,24269410,24278596,24279196,24279540,24279628,24282828,24283093,24283556,24286003,24286019,24287169,24287327,24287795,24288047,24288489,24288912,24290289,24290971,24291857,24292739,24292955,24299747,24390674,24391539,24392268,24392399,24392496,24393382,24394548,24395541,24396645,24396818,24398124,24398996,24400009,24400945,24401557,24406381,24406604,24406983,24407199,24407452,24408951,24411026,39322399,39322504,39322574\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20221101\"},{\"key\":\"client.name\",\"value\":\"WEB\"},{\"key\":\"client.fexp\",\"value\":\"24185614,24036947,24199724,24391539,24268142,24396645,24406604,24164186,24282828,24299747,24187043,24392399,24291857,24219713,24394548,24288489,24152443,24080738,24406983,39322399,24191629,39322504,24140247,24286003,24209350,24265820,23983296,24255545,24254502,24287327,24287169,24400009,24286019,23882502,24279196,24279540,24255543,24396818,23946420,23986016,23918597,24411026,23885487,24279628,24002022,24034168,23966208,24269410,24181174,24169501,24267570,24290971,24161116,24398124,24001373,24292955,24120820,1714257,23804281,24278596,24077241,24262346,24287795,24002025,24248092,24392496,24398996,24400945,24283093,24241378,24406381,24263796,24267564,24392268,24288912,24162920,24260783,24166867,24407452,24184446,24407199,24229161,24230618,24290289,24408951,39322574,23998056,24288047,24211178,24007246,24135310,24401557,24206234,23934970,24393382,24283556,24395541,24004644,24255165,24292739,24246502,24390674,24187377\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"endpoint\":{\"clickTrackingParams\":\"IhMI157_vYWQ-wIVjs5VCh2k1A1NMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/youtubei/v1/navigation/resolve_url\",\"webPageType\":\"WEB_PAGE_TYPE_CHANNEL\",\"rootVe\":3611,\"apiUrl\":\"/youtubei/v1/browse\"},\"resolveUrlCommandMetadata\":{\"isVanityUrl\":true}},\"browseEndpoint\":{\"browseId\":\"UC6nSFpj9HTCZ5t-N3Rm3-HA\",\"params\":\"EgC4AQDyBgQKAjIA\"}}}",
371371
"latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse"
372372
}
373373
}

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_3.json‎

Lines changed: 9 additions & 9 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)