Skip to content

Commit 533121f

Browse files
AudricVAudricV
authored
Merge pull request #1045 from Theta-Dev/fix/trending-video-tab
[YouTube] Extract trends from A/B tested "Videos" tab and fix extraction of trends name from A/B tested new title design
2 parents 92a0024 + 3673d4a commit 533121f

7 files changed

Lines changed: 168 additions & 63 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,15 @@
4141

4242
import java.io.IOException;
4343
import java.nio.charset.StandardCharsets;
44+
import java.util.stream.Stream;
4445

4546
import javax.annotation.Nonnull;
4647

4748
public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
4849
private JsonObject initialData;
4950

51+
private static final String VIDEOS_TAB_PARAMS = "4gIOGgxtb3N0X3BvcHVsYXI%3D";
52+
5053
public YoutubeTrendingExtractor(final StreamingService service,
5154
final ListLinkHandler linkHandler,
5255
final String kioskId) {
@@ -60,6 +63,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
6063
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
6164
getExtractorContentCountry())
6265
.value("browseId", "FEtrending")
66+
.value("params", VIDEOS_TAB_PARAMS)
6367
.done())
6468
.getBytes(StandardCharsets.UTF_8);
6569
// @formatter:on
@@ -81,6 +85,8 @@ public String getName() throws ParsingException {
8185
name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title");
8286
} else if (header.has("c4TabbedHeaderRenderer")) {
8387
name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title");
88+
} else if (header.has("pageHeaderRenderer")) {
89+
name = getTextAtKey(header.getObject("pageHeaderRenderer"), "pageTitle");
8490
}
8591

8692
if (isNullOrEmpty(name)) {
@@ -94,7 +100,10 @@ public String getName() throws ParsingException {
94100
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
95101
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
96102
final TimeAgoParser timeAgoParser = getTimeAgoParser();
97-
final JsonObject tabContent = getTrendingTabContent();
103+
final JsonObject tab = getTrendingTab();
104+
final JsonObject tabContent = tab.getObject("content");
105+
final boolean isVideoTab = tab.getObject("endpoint").getObject("browseEndpoint")
106+
.getString("params", "").equals(VIDEOS_TAB_PARAMS);
98107

99108
if (tabContent.has("richGridRenderer")) {
100109
tabContent.getObject("richGridRenderer")
@@ -110,7 +119,7 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
110119
.forEachOrdered(videoRenderer -> collector.commit(
111120
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
112121
} else if (tabContent.has("sectionListRenderer")) {
113-
tabContent.getObject("sectionListRenderer")
122+
final Stream<JsonObject> shelves = tabContent.getObject("sectionListRenderer")
114123
.getArray("contents")
115124
.stream()
116125
.filter(JsonObject.class::isInstance)
@@ -120,11 +129,19 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
120129
.stream())
121130
.filter(JsonObject.class::isInstance)
122131
.map(JsonObject.class::cast)
123-
.map(content -> content.getObject("shelfRenderer"))
124-
// Filter Trending shorts and Recently trending sections which have a title,
125-
// contrary to normal trends
126-
.filter(shelfRenderer -> !shelfRenderer.has("title"))
127-
.flatMap(shelfRenderer -> shelfRenderer.getObject("content")
132+
.map(content -> content.getObject("shelfRenderer"));
133+
134+
final Stream<JsonObject> items;
135+
if (isVideoTab) {
136+
// The first shelf of the Videos tab contains the normal trends
137+
items = shelves.findFirst().stream();
138+
} else {
139+
// Filter Trending shorts and Recently trending sections which have a title,
140+
// contrary to normal trends
141+
items = shelves.filter(shelfRenderer -> !shelfRenderer.has("title"));
142+
}
143+
144+
items.flatMap(shelfRenderer -> shelfRenderer.getObject("content")
128145
.getObject("expandedShelfContentsRenderer")
129146
.getArray("items")
130147
.stream())
@@ -138,7 +155,7 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
138155
return new InfoItemsPage<>(collector, null);
139156
}
140157

141-
private JsonObject getTrendingTabContent() throws ParsingException {
158+
private JsonObject getTrendingTab() throws ParsingException {
142159
return initialData.getObject("contents")
143160
.getObject("twoColumnBrowseResultsRenderer")
144161
.getArray("tabs")
@@ -150,7 +167,7 @@ private JsonObject getTrendingTabContent() throws ParsingException {
150167
.filter(tabRenderer -> tabRenderer.has("content"))
151168
// There should be at most one tab selected
152169
.findFirst()
153-
.orElseThrow(() -> new ParsingException("Could not get \"Now\" trending tab"))
154-
.getObject("content");
170+
.orElseThrow(() ->
171+
new ParsingException("Could not get \"Now\" or \"Videos\" trending tab"));
155172
}
156173
}

‎extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_0.json‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
"httpMethod": "GET",
44
"url": "https://www.youtube.com/sw.js",
55
"headers": {
6-
"Origin": [
6+
"Referer": [
77
"https://www.youtube.com"
88
],
9-
"Referer": [
9+
"Origin": [
1010
"https://www.youtube.com"
1111
],
1212
"Accept-Language": [
@@ -29,22 +29,25 @@
2929
"https://www.youtube.com"
3030
],
3131
"alt-svc": [
32-
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
32+
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
3333
],
3434
"cache-control": [
3535
"private, max-age\u003d0"
3636
],
3737
"content-type": [
3838
"text/javascript; charset\u003dutf-8"
3939
],
40-
"cross-origin-opener-policy-report-only": [
40+
"cross-origin-opener-policy": [
4141
"same-origin; report-to\u003d\"youtube_main\""
4242
],
4343
"date": [
44-
"Tue, 22 Nov 2022 10:40:53 GMT"
44+
"Sun, 16 Apr 2023 17:42:25 GMT"
4545
],
4646
"expires": [
47-
"Tue, 22 Nov 2022 10:40:53 GMT"
47+
"Sun, 16 Apr 2023 17:42:25 GMT"
48+
],
49+
"origin-trial": [
50+
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
4851
],
4952
"p3p": [
5053
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
@@ -59,9 +62,9 @@
5962
"ESF"
6063
],
6164
"set-cookie": [
62-
"YSC\u003dKWKE6LaMJTE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
63-
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:40:53 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
64-
"CONSENT\u003dPENDING+649; expires\u003dThu, 21-Nov-2024 10:40:53 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
65+
"YSC\u003deNq1o5_KNzg; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
66+
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Jul-2020 17:42:25 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
67+
"CONSENT\u003dPENDING+549; expires\u003dTue, 15-Apr-2025 17:42:25 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
6568
],
6669
"strict-transport-security": [
6770
"max-age\u003d31536000"

‎extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_1.json‎

Lines changed: 10 additions & 7 deletions
Large diffs are not rendered by default.

‎extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_2.json‎

Lines changed: 49 additions & 11 deletions
Large diffs are not rendered by default.

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_0.json‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
"httpMethod": "GET",
44
"url": "https://www.youtube.com/sw.js",
55
"headers": {
6-
"Origin": [
6+
"Referer": [
77
"https://www.youtube.com"
88
],
9-
"Referer": [
9+
"Origin": [
1010
"https://www.youtube.com"
1111
],
1212
"Accept-Language": [
@@ -29,22 +29,25 @@
2929
"https://www.youtube.com"
3030
],
3131
"alt-svc": [
32-
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
32+
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
3333
],
3434
"cache-control": [
3535
"private, max-age\u003d0"
3636
],
3737
"content-type": [
3838
"text/javascript; charset\u003dutf-8"
3939
],
40-
"cross-origin-opener-policy-report-only": [
40+
"cross-origin-opener-policy": [
4141
"same-origin; report-to\u003d\"youtube_main\""
4242
],
4343
"date": [
44-
"Tue, 22 Nov 2022 10:40:26 GMT"
44+
"Sun, 16 Apr 2023 17:35:48 GMT"
4545
],
4646
"expires": [
47-
"Tue, 22 Nov 2022 10:40:26 GMT"
47+
"Sun, 16 Apr 2023 17:35:48 GMT"
48+
],
49+
"origin-trial": [
50+
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
4851
],
4952
"p3p": [
5053
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
@@ -59,9 +62,9 @@
5962
"ESF"
6063
],
6164
"set-cookie": [
62-
"YSC\u003daSSq4mC6HTI; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
63-
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:40:26 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
64-
"CONSENT\u003dPENDING+953; expires\u003dThu, 21-Nov-2024 10:40:26 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
65+
"YSC\u003dKx2AfujDxNk; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
66+
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Jul-2020 17:35:48 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
67+
"CONSENT\u003dPENDING+605; expires\u003dTue, 15-Apr-2025 17:35:48 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
6568
],
6669
"strict-transport-security": [
6770
"max-age\u003d31536000"

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_1.json‎

Lines changed: 9 additions & 6 deletions
Large diffs are not rendered by default.

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_2.json‎

Lines changed: 49 additions & 11 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)