Skip to content

Commit 3d0e302

Browse files
AudricVAudricV
committed
[YouTube] Add base class to parse kiosks from WEB InnerTube client
Kiosks structure work in a very similar way to channel tabs, so YoutubeChannelHelper is used in this abstract class.
1 parent 260ba47 commit 3d0e302

1 file changed

Lines changed: 208 additions & 0 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeDesktopBaseKioskExtractor.java‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
package org.schabi.newpipe.extractor.services.youtube.extractors.kiosk;
2+
3+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
4+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
5+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
6+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
7+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
8+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
9+
10+
import com.grack.nanojson.JsonArray;
11+
import com.grack.nanojson.JsonObject;
12+
import com.grack.nanojson.JsonWriter;
13+
14+
import org.schabi.newpipe.extractor.Page;
15+
import org.schabi.newpipe.extractor.StreamingService;
16+
import org.schabi.newpipe.extractor.downloader.Downloader;
17+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
18+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
19+
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
20+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
21+
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
22+
import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo;
23+
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
24+
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamInfoItemExtractor;
25+
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamInfoItemLockupExtractor;
26+
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
27+
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
28+
29+
import java.io.IOException;
30+
import java.nio.charset.StandardCharsets;
31+
32+
import javax.annotation.Nonnull;
33+
import javax.annotation.Nullable;
34+
35+
abstract class YoutubeDesktopBaseKioskExtractor extends KioskExtractor<StreamInfoItem> {
36+
37+
protected final String browseId;
38+
protected final String params;
39+
40+
protected YoutubeChannelHelper.ChannelResponseData responseData;
41+
42+
protected YoutubeDesktopBaseKioskExtractor(final StreamingService streamingService,
43+
final ListLinkHandler linkHandler,
44+
final String kioskId,
45+
final String browseId,
46+
final String params) {
47+
super(streamingService, linkHandler, kioskId);
48+
this.browseId = browseId;
49+
this.params = params;
50+
}
51+
52+
@Override
53+
public void onFetchPage(@Nonnull final Downloader downloader)
54+
throws IOException, ExtractionException {
55+
responseData = YoutubeChannelHelper.getChannelResponse(
56+
browseId,
57+
params,
58+
getExtractorLocalization(),
59+
getExtractorContentCountry());
60+
}
61+
62+
@Nonnull
63+
@Override
64+
public String getName() throws ParsingException {
65+
return YoutubeChannelHelper.getChannelName(
66+
YoutubeChannelHelper.getChannelHeader(responseData.jsonResponse),
67+
null,
68+
responseData.jsonResponse);
69+
}
70+
71+
@Nonnull
72+
@Override
73+
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
74+
final JsonObject tabRendererContent = responseData.jsonResponse.getObject("contents")
75+
.getObject("twoColumnBrowseResultsRenderer")
76+
.getArray("tabs")
77+
.getObject(0)
78+
.getObject("tabRenderer")
79+
.getObject("content");
80+
81+
final JsonArray tabContents;
82+
if (tabRendererContent.has("sectionListRenderer")) {
83+
tabContents = tabRendererContent.getObject("sectionListRenderer")
84+
.getArray("contents")
85+
.getObject(0)
86+
.getObject("itemSectionRenderer")
87+
.getArray("contents")
88+
.getObject(0)
89+
.getObject("shelfRenderer")
90+
.getObject("content")
91+
.getObject("gridRenderer")
92+
.getArray("items");
93+
} else if (tabRendererContent.has("richGridRenderer")) {
94+
tabContents = tabRendererContent.getObject("richGridRenderer")
95+
.getArray("contents");
96+
} else {
97+
tabContents = new JsonArray();
98+
}
99+
100+
return collectStreamItems(tabContents,
101+
responseData.jsonResponse.getObject("responseContext")
102+
.getString("visitorData"));
103+
}
104+
105+
@Override
106+
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
107+
throws IOException, ExtractionException {
108+
if (page == null || page.getBody() == null) {
109+
throw new IllegalArgumentException("Page is null or doesn't contain a body");
110+
}
111+
112+
final JsonObject continuationResponse = getJsonPostResponse("browse", page.getBody(),
113+
getExtractorLocalization());
114+
115+
final JsonArray continuationItems =
116+
continuationResponse.getArray("onResponseReceivedActions")
117+
.stream()
118+
.filter(JsonObject.class::isInstance)
119+
.map(JsonObject.class::cast)
120+
.filter(jsonObject -> jsonObject.has("appendContinuationItemsAction"))
121+
.map(jsonObject -> jsonObject.getObject("appendContinuationItemsAction"))
122+
.findFirst()
123+
.orElse(new JsonObject())
124+
.getArray("continuationItems");
125+
126+
// The page ID is the visitor data
127+
return collectStreamItems(continuationItems, page.getId());
128+
}
129+
130+
private InfoItemsPage<StreamInfoItem> collectStreamItems(
131+
@Nonnull final JsonArray items,
132+
@Nullable final String visitorData) throws IOException, ExtractionException {
133+
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
134+
135+
final Page nextPage;
136+
if (items.isEmpty()) {
137+
nextPage = null;
138+
} else {
139+
final TimeAgoParser timeAgoParser = getTimeAgoParser();
140+
items.streamAsJsonObjects()
141+
.forEachOrdered(content -> {
142+
if (content.has("richItemRenderer")) {
143+
final JsonObject richItem = content.getObject("richItemRenderer")
144+
.getObject("content");
145+
146+
if (richItem.has("videoRenderer")) {
147+
collector.commit(new YoutubeStreamInfoItemExtractor(
148+
richItem.getObject("videoRenderer"), timeAgoParser));
149+
}
150+
} else if (content.has("gridVideoRenderer")) {
151+
collector.commit(new YoutubeStreamInfoItemExtractor(
152+
content.getObject("gridVideoRenderer"), timeAgoParser));
153+
} else if (content.has("lockupViewModel")) {
154+
// lockupViewModels are not used yet, but may be in the future
155+
final JsonObject lockupViewModel = content.getObject("lockupViewModel");
156+
if ("LOCKUP_CONTENT_TYPE_VIDEO".equals(
157+
lockupViewModel.getString("contentType"))) {
158+
collector.commit(new YoutubeStreamInfoItemLockupExtractor(
159+
lockupViewModel, timeAgoParser));
160+
}
161+
}
162+
});
163+
164+
final JsonObject lastContent = items.getObject(items.size() - 1);
165+
if (lastContent.has("continuationItemRenderer")) {
166+
nextPage = getNextPageFrom(
167+
lastContent.getObject("continuationItemRenderer"), visitorData);
168+
} else {
169+
nextPage = null;
170+
}
171+
}
172+
173+
return new InfoItemsPage<>(collector, nextPage);
174+
}
175+
176+
177+
@Nullable
178+
private Page getNextPageFrom(@Nullable final JsonObject continuation,
179+
@Nullable final String visitorData)
180+
throws IOException, ExtractionException {
181+
if (isNullOrEmpty(continuation)) {
182+
return null;
183+
}
184+
185+
final JsonObject continuationEndpoint = continuation.getObject("continuationEndpoint");
186+
final String continuationToken = continuationEndpoint.getObject("continuationCommand")
187+
.getString("token");
188+
189+
// Visitor data is required to get videos in continuations, so we need to apply it to the
190+
// next page and save it as an ID so it can be applied to future continuations
191+
final InnertubeClientRequestInfo webClientRequestInfo =
192+
InnertubeClientRequestInfo.ofWebClient();
193+
webClientRequestInfo.clientInfo.clientVersion = getClientVersion();
194+
webClientRequestInfo.clientInfo.visitorData = visitorData;
195+
196+
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
197+
getExtractorContentCountry(),
198+
webClientRequestInfo,
199+
null)
200+
.value("continuation", continuationToken)
201+
.done())
202+
.getBytes(StandardCharsets.UTF_8);
203+
204+
// The URL is not needed and used, it is only provided to make Page.isValid return true
205+
return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, visitorData,
206+
null, null, body);
207+
}
208+
}

0 commit comments

Comments
 (0)