Skip to content

Commit 678c98f

Browse files
AudricVAudricV
authored
Merge pull request #1127 from AudricV/yt_improvements-and-fixes
[YouTube] Make some improvements and fixes
2 parents eac850c + ec0194c commit 678c98f

391 files changed

Lines changed: 23181 additions & 12148 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 98 additions & 67 deletions
Large diffs are not rendered by default.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -448,9 +448,6 @@ private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws Parsin
448448
case "playlists":
449449
addNonVideosTab.accept(ChannelTabs.PLAYLISTS);
450450
break;
451-
case "channels":
452-
addNonVideosTab.accept(ChannelTabs.CHANNELS);
453-
break;
454451
}
455452
}
456453
});

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,10 +80,9 @@ private String getChannelTabsParameters() throws ParsingException {
8080
return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D";
8181
case ChannelTabs.PLAYLISTS:
8282
return "EglwbGF5bGlzdHPyBgQKAkIA";
83-
case ChannelTabs.CHANNELS:
84-
return "EghjaGFubmVsc_IGBAoCUgA%3D";
83+
default:
84+
throw new ParsingException("Unsupported channel tab: " + name);
8585
}
86-
throw new ParsingException("Unsupported channel tab: " + name);
8786
}
8887

8988
@Override
@@ -313,9 +312,6 @@ private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector
313312
} else if (item.has("gridPlaylistRenderer")) {
314313
getCommitPlaylistConsumer(collector, channelIds,
315314
item.getObject("gridPlaylistRenderer"));
316-
} else if (item.has("gridChannelRenderer")) {
317-
collector.commit(new YoutubeChannelInfoItemExtractor(
318-
item.getObject("gridChannelRenderer")));
319315
} else if (item.has("shelfRenderer")) {
320316
return collectItem(collector, item.getObject("shelfRenderer")
321317
.getObject("content"), channelIds);

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java‎

Lines changed: 18 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
5454
throws IOException, ExtractionException {
5555
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
5656

57-
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
57+
final String url = "https://music.youtube.com/youtubei/v1/search?key="
5858
+ youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
5959

6060
final String params;
@@ -89,20 +89,18 @@ public void onFetchPage(@Nonnull final Downloader downloader)
8989
.value("clientVersion", youtubeMusicKeys[2])
9090
.value("hl", "en-GB")
9191
.value("gl", getExtractorContentCountry().getCountryCode())
92-
.array("experimentIds").end()
93-
.value("experimentsToken", "")
94-
.object("locationInfo").end()
95-
.object("musicAppInfo").end()
92+
.value("platform", "DESKTOP")
93+
.value("utcOffsetMinutes", 0)
9694
.end()
97-
.object("capabilities").end()
9895
.object("request")
99-
.array("internalExperimentFlags").end()
100-
.object("sessionIndex").end()
96+
.array("internalExperimentFlags")
97+
.end()
98+
.value("useSsl", true)
10199
.end()
102-
.object("activePlayers").end()
103100
.object("user")
104-
// TO DO: provide a way to enable restricted mode with:
105-
.value("enableSafetyMode", false)
101+
// TODO: provide a way to enable restricted mode with:
102+
// .value("enableSafetyMode", boolean)
103+
.value("lockedSafetyMode", false)
106104
.end()
107105
.end()
108106
.value("query", getSearchString())
@@ -219,20 +217,18 @@ public InfoItemsPage<InfoItem> getPage(final Page page)
219217
.value("clientVersion", youtubeMusicKeys[2])
220218
.value("hl", "en-GB")
221219
.value("gl", getExtractorContentCountry().getCountryCode())
222-
.array("experimentIds").end()
223-
.value("experimentsToken", "")
220+
.value("platform", "DESKTOP")
224221
.value("utcOffsetMinutes", 0)
225-
.object("locationInfo").end()
226-
.object("musicAppInfo").end()
227222
.end()
228-
.object("capabilities").end()
229223
.object("request")
230-
.array("internalExperimentFlags").end()
231-
.object("sessionIndex").end()
224+
.array("internalExperimentFlags")
225+
.end()
226+
.value("useSsl", true)
232227
.end()
233-
.object("activePlayers").end()
234228
.object("user")
235-
.value("enableSafetyMode", false)
229+
// TODO: provide a way to enable restricted mode with:
230+
// .value("enableSafetyMode", boolean)
231+
.value("lockedSafetyMode", false)
236232
.end()
237233
.end()
238234
.end().done().getBytes(StandardCharsets.UTF_8);
@@ -310,7 +306,7 @@ private Page getNextPageFrom(final JsonArray continuations)
310306
final String continuation = nextContinuationData.getString("continuation");
311307

312308
return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation
313-
+ "&continuation=" + continuation + "&alt=json" + "&key="
314-
+ YoutubeParsingHelper.getYoutubeMusicKey()[0]);
309+
+ "&continuation=" + continuation + "&key="
310+
+ YoutubeParsingHelper.getYoutubeMusicKey()[0] + DISABLE_PRETTY_PRINT_PARAMETER);
315311
}
316312
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java‎

Lines changed: 34 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
77
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
88
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
9+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ALL;
10+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
11+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
12+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
913
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
1014
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
1115

@@ -32,6 +36,7 @@
3236
import java.io.IOException;
3337
import java.nio.charset.StandardCharsets;
3438
import java.util.List;
39+
import java.util.Objects;
3540

3641
import javax.annotation.Nonnull;
3742
import javax.annotation.Nullable;
@@ -57,28 +62,37 @@
5762
*/
5863

5964
public class YoutubeSearchExtractor extends SearchExtractor {
65+
66+
@Nullable
67+
private final String searchType;
68+
private final boolean extractVideoResults;
69+
private final boolean extractChannelResults;
70+
private final boolean extractPlaylistResults;
71+
6072
private JsonObject initialData;
6173

6274
public YoutubeSearchExtractor(final StreamingService service,
6375
final SearchQueryHandler linkHandler) {
6476
super(service, linkHandler);
77+
final List<String> contentFilters = linkHandler.getContentFilters();
78+
searchType = isNullOrEmpty(contentFilters) ? null : contentFilters.get(0);
79+
// Save whether we should extract video, channel and playlist results depending on the
80+
// requested search type, as YouTube returns sometimes videos inside channel search results
81+
// If no search type is provided or ALL filter is requested, extract everything
82+
extractVideoResults = searchType == null || ALL.equals(searchType)
83+
|| VIDEOS.equals(searchType);
84+
extractChannelResults = searchType == null || ALL.equals(searchType)
85+
|| CHANNELS.equals(searchType);
86+
extractPlaylistResults = searchType == null || ALL.equals(searchType)
87+
|| PLAYLISTS.equals(searchType);
6588
}
6689

6790
@Override
6891
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
6992
ExtractionException {
7093
final String query = super.getSearchString();
7194
final Localization localization = getExtractorLocalization();
72-
73-
// Get the search parameter of the request
74-
final List<String> contentFilters = super.getLinkHandler().getContentFilters();
75-
final String params;
76-
if (!isNullOrEmpty(contentFilters)) {
77-
final String searchType = contentFilters.get(0);
78-
params = getSearchParameter(searchType);
79-
} else {
80-
params = "";
81-
}
95+
final String params = getSearchParameter(searchType);
8296

8397
final JsonBuilder<JsonObject> jsonBody = prepareDesktopJsonBuilder(localization,
8498
getExtractorContentCountry())
@@ -111,18 +125,17 @@ public String getSearchSuggestion() throws ParsingException {
111125
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
112126
.getObject(0)
113127
.getObject("didYouMeanRenderer");
114-
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents")
115-
.getObject(0)
116-
.getObject("showingResultsForRenderer");
117128

118129
if (!didYouMeanRenderer.isEmpty()) {
119130
return JsonUtils.getString(didYouMeanRenderer,
120131
"correctedQueryEndpoint.searchEndpoint.query");
121-
} else if (showingResultsForRenderer != null) {
122-
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
123-
} else {
124-
return "";
125132
}
133+
134+
return Objects.requireNonNullElse(
135+
getTextFromObject(itemSectionRenderer.getArray("contents")
136+
.getObject(0)
137+
.getObject("showingResultsForRenderer")
138+
.getObject("correctedQuery")), "");
126139
}
127140

128141
@Override
@@ -211,7 +224,7 @@ public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,
211224

212225
private void collectStreamsFrom(final MultiInfoItemsCollector collector,
213226
@Nonnull final JsonArray contents)
214-
throws NothingFoundException, ParsingException {
227+
throws NothingFoundException {
215228
final TimeAgoParser timeAgoParser = getTimeAgoParser();
216229

217230
for (final Object content : contents) {
@@ -220,13 +233,13 @@ private void collectStreamsFrom(final MultiInfoItemsCollector collector,
220233
throw new NothingFoundException(
221234
getTextFromObject(item.getObject("backgroundPromoRenderer")
222235
.getObject("bodyText")));
223-
} else if (item.has("videoRenderer")) {
236+
} else if (extractVideoResults && item.has("videoRenderer")) {
224237
collector.commit(new YoutubeStreamInfoItemExtractor(
225238
item.getObject("videoRenderer"), timeAgoParser));
226-
} else if (item.has("channelRenderer")) {
239+
} else if (extractChannelResults && item.has("channelRenderer")) {
227240
collector.commit(new YoutubeChannelInfoItemExtractor(
228241
item.getObject("channelRenderer")));
229-
} else if (item.has("playlistRenderer")) {
242+
} else if (extractPlaylistResults && item.has("playlistRenderer")) {
230243
collector.commit(new YoutubePlaylistInfoItemExtractor(
231244
item.getObject("playlistRenderer")));
232245
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 79 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -388,57 +388,43 @@ public long getLikeCount() throws ParsingException {
388388

389389
// If ratings are not allowed, there is no like count available
390390
if (!playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
391-
return -1;
391+
return -1L;
392392
}
393393

394-
String likesString = "";
394+
final JsonArray topLevelButtons = getVideoPrimaryInfoRenderer()
395+
.getObject("videoActions")
396+
.getObject("menuRenderer")
397+
.getArray("topLevelButtons");
395398

396399
try {
397-
final JsonArray topLevelButtons = getVideoPrimaryInfoRenderer()
398-
.getObject("videoActions")
399-
.getObject("menuRenderer")
400-
.getArray("topLevelButtons");
400+
return parseLikeCountFromLikeButtonViewModel(topLevelButtons);
401+
} catch (final ParsingException ignored) {
402+
// A segmentedLikeDislikeButtonRenderer could be returned instead of a
403+
// segmentedLikeDislikeButtonViewModel, so ignore extraction errors relative to
404+
// segmentedLikeDislikeButtonViewModel object
405+
}
401406

402-
// Try first with the new video actions buttons data structure
403-
JsonObject likeToggleButtonRenderer = topLevelButtons.stream()
404-
.filter(JsonObject.class::isInstance)
405-
.map(JsonObject.class::cast)
406-
.map(button -> button.getObject("segmentedLikeDislikeButtonRenderer")
407-
.getObject("likeButton")
408-
.getObject("toggleButtonRenderer"))
409-
.filter(toggleButtonRenderer -> !isNullOrEmpty(toggleButtonRenderer))
410-
.findFirst()
411-
.orElse(null);
412-
413-
// Use the old video actions buttons data structure if the new one isn't returned
414-
if (likeToggleButtonRenderer == null) {
415-
/*
416-
In the old video actions buttons data structure, there are 3 ways to detect whether
417-
a button is the like button, using its toggleButtonRenderer:
418-
- checking whether toggleButtonRenderer.targetId is equal to watch-like;
419-
- checking whether toggleButtonRenderer.defaultIcon.iconType is equal to LIKE;
420-
- checking whether
421-
toggleButtonRenderer.toggleButtonSupportedData.toggleButtonIdData.id
422-
is equal to TOGGLE_BUTTON_ID_TYPE_LIKE.
423-
*/
424-
likeToggleButtonRenderer = topLevelButtons.stream()
425-
.filter(JsonObject.class::isInstance)
426-
.map(JsonObject.class::cast)
427-
.map(topLevelButton -> topLevelButton.getObject("toggleButtonRenderer"))
428-
.filter(toggleButtonRenderer -> toggleButtonRenderer.getString("targetId")
429-
.equalsIgnoreCase("watch-like")
430-
|| toggleButtonRenderer.getObject("defaultIcon")
431-
.getString("iconType")
432-
.equalsIgnoreCase("LIKE")
433-
|| toggleButtonRenderer.getObject("toggleButtonSupportedData")
434-
.getObject("toggleButtonIdData")
435-
.getString("id")
436-
.equalsIgnoreCase("TOGGLE_BUTTON_ID_TYPE_LIKE"))
437-
.findFirst()
438-
.orElseThrow(() -> new ParsingException(
439-
"The like button is missing even though ratings are enabled"));
440-
}
407+
try {
408+
return parseLikeCountFromLikeButtonRenderer(topLevelButtons);
409+
} catch (final ParsingException e) {
410+
throw new ParsingException("Could not get like count", e);
411+
}
412+
}
441413

414+
private static long parseLikeCountFromLikeButtonRenderer(
415+
@Nonnull final JsonArray topLevelButtons) throws ParsingException {
416+
String likesString = null;
417+
final JsonObject likeToggleButtonRenderer = topLevelButtons.stream()
418+
.filter(JsonObject.class::isInstance)
419+
.map(JsonObject.class::cast)
420+
.map(button -> button.getObject("segmentedLikeDislikeButtonRenderer")
421+
.getObject("likeButton")
422+
.getObject("toggleButtonRenderer"))
423+
.filter(toggleButtonRenderer -> !isNullOrEmpty(toggleButtonRenderer))
424+
.findFirst()
425+
.orElse(null);
426+
427+
if (likeToggleButtonRenderer != null) {
442428
// Use one of the accessibility strings available (this one has the same path as the
443429
// one used for comments' like count extraction)
444430
likesString = likeToggleButtonRenderer.getObject("accessibilityData")
@@ -460,23 +446,58 @@ public long getLikeCount() throws ParsingException {
460446
.getString("label");
461447
}
462448

463-
// If ratings are allowed and the likes string is null, it means that we couldn't
464-
// extract the (real) like count from accessibility data
465-
if (likesString == null) {
466-
throw new ParsingException("Could not get like count from accessibility data");
467-
}
468-
469449
// This check only works with English localizations!
470-
if (likesString.toLowerCase().contains("no likes")) {
450+
if (likesString != null && likesString.toLowerCase().contains("no likes")) {
471451
return 0;
472452
}
453+
}
473454

474-
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
475-
} catch (final NumberFormatException nfe) {
476-
throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer",
477-
nfe);
478-
} catch (final Exception e) {
479-
throw new ParsingException("Could not get like count", e);
455+
// If ratings are allowed and the likes string is null, it means that we couldn't extract
456+
// the full like count from accessibility data
457+
if (likesString == null) {
458+
throw new ParsingException("Could not get like count from accessibility data");
459+
}
460+
461+
try {
462+
return Long.parseLong(Utils.removeNonDigitCharacters(likesString));
463+
} catch (final NumberFormatException e) {
464+
throw new ParsingException("Could not parse \"" + likesString + "\" as a long", e);
465+
}
466+
}
467+
468+
private static long parseLikeCountFromLikeButtonViewModel(
469+
@Nonnull final JsonArray topLevelButtons) throws ParsingException {
470+
// Try first with the current video actions buttons data structure
471+
final JsonObject likeToggleButtonViewModel = topLevelButtons.stream()
472+
.filter(JsonObject.class::isInstance)
473+
.map(JsonObject.class::cast)
474+
.map(button -> button.getObject("segmentedLikeDislikeButtonViewModel")
475+
.getObject("likeButtonViewModel")
476+
.getObject("likeButtonViewModel")
477+
.getObject("toggleButtonViewModel")
478+
.getObject("toggleButtonViewModel")
479+
.getObject("defaultButtonViewModel")
480+
.getObject("buttonViewModel"))
481+
.filter(buttonViewModel -> !isNullOrEmpty(buttonViewModel))
482+
.findFirst()
483+
.orElse(null);
484+
485+
if (likeToggleButtonViewModel == null) {
486+
throw new ParsingException("Could not find buttonViewModel object");
487+
}
488+
489+
final String accessibilityText = likeToggleButtonViewModel.getString("accessibilityText");
490+
if (accessibilityText == null) {
491+
throw new ParsingException("Could not find buttonViewModel's accessibilityText string");
492+
}
493+
494+
// The like count is always returned as a number in this element, even for videos with no
495+
// likes
496+
try {
497+
return Long.parseLong(Utils.removeNonDigitCharacters(accessibilityText));
498+
} catch (final NumberFormatException e) {
499+
throw new ParsingException(
500+
"Could not parse \"" + accessibilityText + "\" as a long", e);
480501
}
481502
}
482503

0 commit comments

Comments
 (0)