Skip to content

Commit 5a101fd

Browse files
TobiGrTobiGr
authored
Merge pull request #262 from wb9688/pbj
Improve yt_new
2 parents fc465c8 + 2807079 commit 5a101fd

17 files changed

Lines changed: 605 additions & 554 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 50 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,27 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5-
import com.grack.nanojson.JsonParser;
6-
import com.grack.nanojson.JsonParserException;
75

8-
import org.jsoup.nodes.Document;
96
import org.schabi.newpipe.extractor.StreamingService;
107
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
118
import org.schabi.newpipe.extractor.downloader.Downloader;
12-
import org.schabi.newpipe.extractor.downloader.Response;
139
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
1410
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1511
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
1612
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
13+
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
1714
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
1815
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1916
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
2017
import org.schabi.newpipe.extractor.utils.Utils;
2118

2219
import java.io.IOException;
23-
import java.util.Collections;
24-
import java.util.HashMap;
25-
import java.util.List;
26-
import java.util.Map;
2720

2821
import javax.annotation.Nonnull;
2922

30-
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
31-
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
23+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
24+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
25+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
3226

3327
/*
3428
* Created by Christian Schabesberger on 25.07.16.
@@ -52,35 +46,36 @@
5246

5347
@SuppressWarnings("WeakerAccess")
5448
public class YoutubeChannelExtractor extends ChannelExtractor {
55-
/*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
56-
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
57-
58-
private Document doc;
5949
private JsonObject initialData;
50+
private JsonObject videoTab;
6051

6152
public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
6253
super(service, linkHandler);
6354
}
6455

6556
@Override
6657
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
67-
String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS;
68-
final Response response = downloader.get(channelUrl, getExtractorLocalization());
69-
doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response);
70-
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
58+
final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
59+
60+
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
61+
62+
initialData = ajaxJson.getObject(1).getObject("response");
7163
}
7264

7365

7466
@Override
7567
public String getNextPageUrl() throws ExtractionException {
76-
return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("continuations"));
68+
if (getVideoTab() == null) return "";
69+
return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer")
70+
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
71+
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("continuations"));
7772
}
7873

7974
@Nonnull
8075
@Override
8176
public String getUrl() throws ParsingException {
8277
try {
83-
return CHANNEL_URL_BASE + getId();
78+
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + getId());
8479
} catch (ParsingException e) {
8580
return super.getUrl();
8681
}
@@ -109,8 +104,10 @@ public String getName() throws ParsingException {
109104
@Override
110105
public String getAvatarUrl() throws ParsingException {
111106
try {
112-
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar")
107+
String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar")
113108
.getArray("thumbnails").getObject(0).getString("url");
109+
110+
return fixThumbnailUrl(url);
114111
} catch (Exception e) {
115112
throw new ParsingException("Could not get avatar", e);
116113
}
@@ -127,17 +124,8 @@ public String getBannerUrl() throws ParsingException {
127124
if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) {
128125
return null;
129126
}
130-
// the first characters of the banner URLs are different for each channel and some are not even valid URLs
131-
if (url.startsWith("//")) {
132-
url = url.substring(2);
133-
}
134-
if (url.startsWith(HTTP)) {
135-
url = Utils.replaceHttpWithHttps(url);
136-
} else if (!url.startsWith(HTTPS)) {
137-
url = HTTPS + url;
138-
}
139127

140-
return url;
128+
return fixThumbnailUrl(url);
141129
} catch (Exception e) {
142130
throw new ParsingException("Could not get banner", e);
143131
}
@@ -157,13 +145,17 @@ public long getSubscriberCount() throws ParsingException {
157145
final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText");
158146
if (subscriberInfo != null) {
159147
try {
160-
return Utils.mixedNumberWordToLong(subscriberInfo.getArray("runs").getObject(0).getString("text"));
148+
return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo));
161149
} catch (NumberFormatException e) {
162150
throw new ParsingException("Could not get subscriber count", e);
163151
}
164152
} else {
165-
// If the element is null, the channel have the subscriber count disabled
166-
return -1;
153+
// If there's no subscribe button, the channel has the subscriber count disabled
154+
if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) {
155+
return -1;
156+
} else {
157+
return 0;
158+
}
167159
}
168160
}
169161

@@ -181,8 +173,12 @@ public String getDescription() throws ParsingException {
181173
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
182174
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
183175

184-
JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents");
185-
collectStreamsFrom(collector, videos);
176+
if (getVideoTab() != null) {
177+
JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents")
178+
.getObject(0).getObject("itemSectionRenderer").getArray("contents").getObject(0)
179+
.getObject("gridRenderer").getArray("items");
180+
collectStreamsFrom(collector, videos);
181+
}
186182

187183
return new InfoItemsPage<>(collector, getNextPageUrl());
188184
}
@@ -198,46 +194,19 @@ public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException,
198194
fetchPage();
199195

200196
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
201-
JsonArray ajaxJson;
202-
203-
Map<String, List<String>> headers = new HashMap<>();
204-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
205-
try {
206-
// Use the hardcoded client version first to get JSON with a structure we know
207-
headers.put("X-YouTube-Client-Version",
208-
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
209-
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
210-
if (response.length() < 50) { // ensure to have a valid response
211-
throw new ParsingException("Could not parse json data for next streams");
212-
}
213-
ajaxJson = JsonParser.array().from(response);
214-
} catch (Exception e) {
215-
try {
216-
headers.put("X-YouTube-Client-Version",
217-
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
218-
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
219-
if (response.length() < 50) { // ensure to have a valid response
220-
throw new ParsingException("Could not parse json data for next streams");
221-
}
222-
ajaxJson = JsonParser.array().from(response);
223-
} catch (JsonParserException ignored) {
224-
throw new ParsingException("Could not parse json data for next streams", e);
225-
}
226-
}
197+
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
227198

228199
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
229-
.getObject("continuationContents").getObject("sectionListContinuation");
200+
.getObject("continuationContents").getObject("gridContinuation");
230201

231-
collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));
202+
collectStreamsFrom(collector, sectionListContinuation.getArray("items"));
232203

233204
return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
234205
}
235206

236207

237208
private String getNextPageUrlFrom(JsonArray continuations) {
238-
if (continuations == null) {
239-
return "";
240-
}
209+
if (continuations == null) return "";
241210

242211
JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
243212
String continuation = nextContinuationData.getString("continuation");
@@ -254,10 +223,9 @@ private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray vi
254223
final TimeAgoParser timeAgoParser = getTimeAgoParser();
255224

256225
for (Object video : videos) {
257-
JsonObject videoInfo = ((JsonObject) video).getObject("itemSectionRenderer")
258-
.getArray("contents").getObject(0);
259-
if (videoInfo.getObject("videoRenderer") != null) {
260-
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo.getObject("videoRenderer"), timeAgoParser) {
226+
if (((JsonObject) video).getObject("gridVideoRenderer") != null) {
227+
collector.commit(new YoutubeStreamInfoItemExtractor(
228+
((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) {
261229
@Override
262230
public String getUploaderName() {
263231
return uploaderName;
@@ -273,6 +241,8 @@ public String getUploaderUrl() {
273241
}
274242

275243
private JsonObject getVideoTab() throws ParsingException {
244+
if (this.videoTab != null) return this.videoTab;
245+
276246
JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
277247
.getArray("tabs");
278248
JsonObject videoTab = null;
@@ -290,6 +260,15 @@ private JsonObject getVideoTab() throws ParsingException {
290260
throw new ParsingException("Could not find Videos tab");
291261
}
292262

263+
try {
264+
if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
265+
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
266+
.getArray("contents").getObject(0).getObject("messageRenderer")
267+
.getObject("text")).equals("This channel has no videos."))
268+
return null;
269+
} catch (Exception ignored) {}
270+
271+
this.videoTab = videoTab;
293272
return videoTab;
294273
}
295274
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java‎

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
88
import org.schabi.newpipe.extractor.utils.Utils;
99

10-
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
11-
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
10+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
11+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
1212

1313
/*
1414
* Created by Christian Schabesberger on 12.02.17.
@@ -41,15 +41,8 @@ public YoutubeChannelInfoItemExtractor(JsonObject channelInfoItem) {
4141
public String getThumbnailUrl() throws ParsingException {
4242
try {
4343
String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
44-
if (url.startsWith("//")) {
45-
url = url.substring(2);
46-
}
47-
if (url.startsWith(HTTP)) {
48-
url = Utils.replaceHttpWithHttps(url);
49-
} else if (!url.startsWith(HTTPS)) {
50-
url = HTTPS + url;
51-
}
52-
return url;
44+
45+
return fixThumbnailUrl(url);
5346
} catch (Exception e) {
5447
throw new ParsingException("Could not get thumbnail url", e);
5548
}
@@ -58,7 +51,7 @@ public String getThumbnailUrl() throws ParsingException {
5851
@Override
5952
public String getName() throws ParsingException {
6053
try {
61-
return channelInfoItem.getObject("title").getString("simpleText");
54+
return getTextFromObject(channelInfoItem.getObject("title"));
6255
} catch (Exception e) {
6356
throw new ParsingException("Could not get name", e);
6457
}
@@ -67,7 +60,7 @@ public String getName() throws ParsingException {
6760
@Override
6861
public String getUrl() throws ParsingException {
6962
try {
70-
String id = "channel/" + channelInfoItem.getString("channelId"); // Does prepending 'channel/' always work?
63+
String id = "channel/" + channelInfoItem.getString("channelId");
7164
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id);
7265
} catch (Exception e) {
7366
throw new ParsingException("Could not get url", e);
@@ -77,7 +70,7 @@ public String getUrl() throws ParsingException {
7770
@Override
7871
public long getSubscriberCount() throws ParsingException {
7972
try {
80-
String subscribers = channelInfoItem.getObject("subscriberCountText").getString("simpleText").split(" ")[0];
73+
String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText"));
8174
return Utils.mixedNumberWordToLong(subscribers);
8275
} catch (Exception e) {
8376
throw new ParsingException("Could not get subscriber count", e);
@@ -87,8 +80,7 @@ public long getSubscriberCount() throws ParsingException {
8780
@Override
8881
public long getStreamCount() throws ParsingException {
8982
try {
90-
return Long.parseLong(Utils.removeNonDigitCharacters(channelInfoItem.getObject("videoCountText")
91-
.getArray("runs").getObject(0).getString("text")));
83+
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText"))));
9284
} catch (Exception e) {
9385
throw new ParsingException("Could not get stream count", e);
9486
}
@@ -97,7 +89,7 @@ public long getStreamCount() throws ParsingException {
9789
@Override
9890
public String getDescription() throws ParsingException {
9991
try {
100-
return channelInfoItem.getObject("descriptionSnippet").getArray("runs").getObject(0).getString("text");
92+
return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
10193
} catch (Exception e) {
10294
throw new ParsingException("Could not get description", e);
10395
}

0 commit comments

Comments
 (0)