Skip to content

Commit 43e9afb

Browse files
Isira-SeneviratneIsira-Seneviratne
committed
[YouTube] Update getUrlFromNavigationEndpoint to return Optional, add helper method for extracting music URL
1 parent 57bd120 commit 43e9afb

12 files changed

Lines changed: 173 additions & 210 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,8 @@ private static void addAllCommandRuns(
237237
return;
238238
}
239239

240-
final String url = getUrlFromNavigationEndpoint(navigationEndpoint);
240+
final String url = getUrlFromNavigationEndpoint(navigationEndpoint)
241+
.orElse(null);
241242
if (url == null) {
242243
return;
243244
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java‎

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -73,18 +73,17 @@ private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelC
7373
metaInfo.setContent(new Description(description, Description.HTML));
7474
if (infoPanelContentRenderer.has("sourceEndpoint")) {
7575
final String metaInfoLinkUrl = getUrlFromNavigationEndpoint(
76-
infoPanelContentRenderer.getObject("sourceEndpoint"));
76+
infoPanelContentRenderer.getObject("sourceEndpoint"))
77+
.orElse("");
7778
try {
7879
metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(
7980
metaInfoLinkUrl))));
8081
} catch (final NullPointerException | MalformedURLException e) {
8182
throw new ParsingException("Could not get metadata info URL", e);
8283
}
8384

84-
final String metaInfoLinkText = getTextFromObject(
85-
infoPanelContentRenderer.getObject("inlineSource"))
86-
.orElseThrow(() ->
87-
new ParsingException("Could not get metadata info link text."));
85+
final var source = infoPanelContentRenderer.getObject("inlineSource");
86+
final String metaInfoLinkText = getTextFromObjectOrThrow(source, "metadata info link");
8887
metaInfo.addUrlText(metaInfoLinkText);
8988
}
9089

@@ -96,12 +95,10 @@ private static MetaInfo getClarificationRenderer(
9695
@Nonnull final JsonObject clarificationRenderer) throws ParsingException {
9796
final MetaInfo metaInfo = new MetaInfo();
9897

99-
final String title = getTextFromObject(clarificationRenderer.getObject("contentTitle"))
100-
.orElseThrow(() ->
101-
new ParsingException("Could not extract clarification renderer content"));
102-
final String text = getTextFromObject(clarificationRenderer.getObject("text"))
103-
.orElseThrow(() ->
104-
new ParsingException("Could not extract clarification renderer content"));
98+
final String title = getTextFromObjectOrThrow(
99+
clarificationRenderer.getObject("contentTitle"), "clarification renderer");
100+
final String text = getTextFromObjectOrThrow(
101+
clarificationRenderer.getObject("text"), "clarification renderer");
105102
metaInfo.setTitle(title);
106103
metaInfo.setContent(new Description(text, Description.PLAIN_TEXT));
107104

@@ -110,22 +107,21 @@ private static MetaInfo getClarificationRenderer(
110107
.getObject("buttonRenderer");
111108
try {
112109
final String url = getUrlFromNavigationEndpoint(actionButton
113-
.getObject("command"));
110+
.getObject("command")).orElse("");
114111
metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url))));
115112
} catch (final NullPointerException | MalformedURLException e) {
116113
throw new ParsingException("Could not get metadata info URL", e);
117114
}
118115

119-
final String metaInfoLinkText = getTextFromObject(actionButton.getObject("text"))
120-
.orElseThrow(() ->
121-
new ParsingException("Could not get metadata info link text."));
122-
metaInfo.addUrlText(metaInfoLinkText);
116+
final String linkText = getTextFromObjectOrThrow(actionButton.getObject("text"),
117+
"metadata info link");
118+
metaInfo.addUrlText(linkText);
123119
}
124120

125121
if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer
126122
.has("secondarySource")) {
127123
final String url = getUrlFromNavigationEndpoint(clarificationRenderer
128-
.getObject("secondaryEndpoint"));
124+
.getObject("secondaryEndpoint")).orElse(null);
129125
// Ignore Google URLs, because those point to a Google search about "Covid-19"
130126
if (url != null && !isGoogleURL(url)) {
131127
try {
@@ -193,10 +189,9 @@ private static void getEmergencyOneboxRenderer(
193189
metaInfo.addUrlText(urlText);
194190

195191
// usually the webpage of the association
196-
final String url = getUrlFromNavigationEndpoint(r.getObject("navigationEndpoint"));
197-
if (url == null) {
198-
throw new ParsingException("Could not extract emergency renderer url");
199-
}
192+
final String url = getUrlFromNavigationEndpoint(r.getObject("navigationEndpoint"))
193+
.orElseThrow(() ->
194+
new ParsingException("Could not extract emergency renderer url"));
200195

201196
try {
202197
metaInfo.addUrl(new URL(replaceHttpWithHttps(url)));

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 97 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -687,87 +687,103 @@ public static String getYoutubeMusicClientVersion()
687687
return youtubeMusicClientVersion;
688688
}
689689

690-
@Nullable
691-
public static String getUrlFromNavigationEndpoint(
690+
@Nonnull
691+
public static Optional<String> getUrlFromNavigationEndpoint(
692692
@Nonnull final JsonObject navigationEndpoint) {
693-
if (navigationEndpoint.has("urlEndpoint")) {
694-
String internUrl = navigationEndpoint.getObject("urlEndpoint")
695-
.getString("url");
696-
if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
697-
// remove https://www.youtube.com part to fall in the next if block
698-
internUrl = internUrl.substring(23);
699-
}
700-
701-
if (internUrl.startsWith("/redirect?")) {
702-
// q parameter can be the first parameter
703-
internUrl = internUrl.substring(10);
704-
final String[] params = internUrl.split("&");
705-
for (final String param : params) {
706-
if (param.split("=")[0].equals("q")) {
707-
return Utils.decodeUrlUtf8(param.split("=")[1]);
693+
return Optional.ofNullable(navigationEndpoint.getObject("urlEndpoint")
694+
.getString("url"))
695+
.map(internUrl -> {
696+
if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
697+
// remove https://www.youtube.com part to fall in the next if block
698+
internUrl = internUrl.substring(23);
708699
}
709-
}
710-
} else if (internUrl.startsWith("http")) {
711-
return internUrl;
712-
} else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user")
713-
|| internUrl.startsWith("/watch")) {
714-
return "https://www.youtube.com" + internUrl;
715-
}
716-
}
717-
718-
if (navigationEndpoint.has("browseEndpoint")) {
719-
final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint");
720-
final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl");
721-
final String browseId = browseEndpoint.getString("browseId");
722-
723-
if (browseId != null) {
724-
if (browseId.startsWith("UC")) {
725-
// All channel IDs are prefixed with UC
726-
return "https://www.youtube.com/channel/" + browseId;
727-
} else if (browseId.startsWith("VL")) {
728-
// All playlist IDs are prefixed with VL, which needs to be removed from the
729-
// playlist ID
730-
return "https://www.youtube.com/playlist?list=" + browseId.substring(2);
731-
}
732-
}
733700

734-
if (!isNullOrEmpty(canonicalBaseUrl)) {
735-
return "https://www.youtube.com" + canonicalBaseUrl;
736-
}
737-
}
738-
739-
if (navigationEndpoint.has("watchEndpoint")) {
740-
final StringBuilder url = new StringBuilder();
741-
url.append("https://www.youtube.com/watch?v=")
742-
.append(navigationEndpoint.getObject("watchEndpoint")
743-
.getString(VIDEO_ID));
744-
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
745-
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint")
746-
.getString("playlistId"));
747-
}
748-
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) {
749-
url.append("&t=")
750-
.append(navigationEndpoint.getObject("watchEndpoint")
751-
.getInt("startTimeSeconds"));
752-
}
753-
return url.toString();
754-
}
755-
756-
if (navigationEndpoint.has("watchPlaylistEndpoint")) {
757-
return "https://www.youtube.com/playlist?list="
758-
+ navigationEndpoint.getObject("watchPlaylistEndpoint")
759-
.getString("playlistId");
760-
}
701+
if (internUrl.startsWith("/redirect?")) {
702+
// q parameter can be the first parameter
703+
internUrl = internUrl.substring(10);
704+
final String[] params = internUrl.split("&");
705+
for (final String param : params) {
706+
final String[] nameAndValue = param.split("=");
707+
if (nameAndValue[0].equals("q")) {
708+
return Utils.decodeUrlUtf8(nameAndValue[1]);
709+
}
710+
}
711+
} else if (internUrl.startsWith("http")) {
712+
return internUrl;
713+
} else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user")
714+
|| internUrl.startsWith("/watch")) {
715+
return "https://www.youtube.com" + internUrl;
716+
}
761717

762-
if (navigationEndpoint.has("commandMetadata")) {
763-
final JsonObject metadata = navigationEndpoint.getObject("commandMetadata")
764-
.getObject("webCommandMetadata");
765-
if (metadata.has("url")) {
766-
return "https://www.youtube.com" + metadata.getString("url");
767-
}
768-
}
718+
return null;
719+
})
720+
.or(() -> {
721+
final var browseEndpoint = navigationEndpoint.getObject("browseEndpoint");
722+
final var baseUrl = browseEndpoint.getString("canonicalBaseUrl");
723+
final var browseId = browseEndpoint.getString("browseId");
724+
725+
return Optional.ofNullable(browseId)
726+
.map(id -> {
727+
if (id.startsWith("UC")) {
728+
// All channel IDs are prefixed with UC
729+
return "https://www.youtube.com/channel/" + id;
730+
} else if (id.startsWith("VL")) {
731+
// All playlist IDs are prefixed with VL, which needs to be
732+
// removed from the playlist ID
733+
return "https://www.youtube.com/playlist?list="
734+
+ id.substring(2);
735+
}
736+
return null;
737+
})
738+
.or(() -> {
739+
if (!isNullOrEmpty(baseUrl)) {
740+
return Optional.of("https://www.youtube.com" + baseUrl);
741+
} else {
742+
return Optional.empty();
743+
}
744+
});
745+
})
746+
.or(() -> {
747+
final var watchEndpoint = navigationEndpoint.getObject("watchEndpoint");
748+
final var videoId = watchEndpoint.getString(VIDEO_ID);
749+
final var playlistId = watchEndpoint.getString("playlistId");
750+
final var startTime = watchEndpoint.getInt("startTimeSeconds", -1);
751+
final String url = "https://www.youtube.com/watch?v=" + videoId
752+
+ (playlistId != null ? "&list=" + playlistId : "")
753+
+ (startTime != -1 ? "&t=" + startTime : "");
754+
return Optional.of(url);
755+
})
756+
.or(() -> {
757+
final var playlistId = navigationEndpoint.getObject("watchPlaylistEndpoint")
758+
.getString("playlistId");
759+
return Optional.ofNullable(playlistId)
760+
.map(id -> "https://www.youtube.com/playlist?list=" + id);
761+
})
762+
.or(() -> {
763+
final var metadata = navigationEndpoint.getObject("commandMetadata")
764+
.getObject("webCommandMetadata");
765+
return Optional.ofNullable(metadata.getString("url"))
766+
.map(url -> "https://www.youtube.com" + url);
767+
})
768+
.filter(url -> !url.isEmpty());
769+
}
769770

770-
return null;
771+
@Nonnull
772+
public static Optional<String> getMusicUploaderUrlFromMenu(@Nonnull final JsonObject object) {
773+
final var items = object.getObject("menu")
774+
.getObject("menuRenderer")
775+
.getArray("items");
776+
return items.streamAsJsonObjects()
777+
.flatMap(item -> {
778+
final var renderer = item.getObject("menuNavigationItemRenderer");
779+
final var iconType = renderer.getObject("icon").getString("iconType");
780+
final var endpoint = renderer.getObject("navigationEndpoint");
781+
782+
return "ARTIST".equals(iconType)
783+
? getUrlFromNavigationEndpoint(endpoint).stream()
784+
: Stream.empty();
785+
})
786+
.findFirst();
771787
}
772788

773789
/**
@@ -790,8 +806,8 @@ public static Optional<String> getTextFromObject(@Nonnull final JsonObject textO
790806

791807
if (html) {
792808
final String url = getUrlFromNavigationEndpoint(
793-
run.getObject("navigationEndpoint"));
794-
if (!isNullOrEmpty(url)) {
809+
run.getObject("navigationEndpoint")).orElse(null);
810+
if (url != null) {
795811
textString = "<a href=\"" + Entities.escape(url) + "\">"
796812
+ Entities.escape(textString) + "</a>";
797813
}
@@ -838,9 +854,9 @@ public static Optional<String> getTextFromObject(@Nonnull final JsonObject textO
838854
@Nonnull
839855
public static Optional<String> getUrlFromObject(@Nonnull final JsonObject textObject) {
840856
return textObject.getArray("runs").streamAsJsonObjects()
841-
.map(textPart -> getUrlFromNavigationEndpoint(textPart
842-
.getObject("navigationEndpoint")))
843-
.filter(url -> !isNullOrEmpty(url))
857+
.flatMap(textPart -> getUrlFromNavigationEndpoint(textPart
858+
.getObject("navigationEndpoint"))
859+
.stream())
844860
.findFirst();
845861
}
846862

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import javax.annotation.Nonnull;
1010
import java.util.List;
1111

12-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
12+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow;
1313
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem;
1414
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
1515

@@ -32,7 +32,8 @@ public String getName() throws ParsingException {
3232

3333
@Override
3434
public String getUrl() throws ParsingException {
35-
return getUrlFromNavigationEndpoint(showRenderer.getObject("navigationEndpoint"));
35+
return getUrlFromNavigationEndpoint(showRenderer.getObject("navigationEndpoint"))
36+
.orElse(null);
3637
}
3738

3839
@Nonnull
@@ -46,11 +47,10 @@ public List<Image> getThumbnails() throws ParsingException {
4647
public long getStreamCount() throws ParsingException {
4748
// The stream count should be always returned in the first text object for English
4849
// localizations, but the complete text is parsed for reliability purposes
49-
final String streamCountText = getTextFromObject(
50-
showRenderer.getObject("thumbnailOverlays")
51-
.getObject("thumbnailOverlayBottomPanelRenderer")
52-
.getObject("text"))
53-
.orElseThrow(() -> new ParsingException("Could not get stream count"));
50+
final var textObject = showRenderer.getObject("thumbnailOverlays")
51+
.getObject("thumbnailOverlayBottomPanelRenderer")
52+
.getObject("text");
53+
final String streamCountText = getTextFromObjectOrThrow(textObject, "stream count");
5454

5555
try {
5656
// The data returned could be a human/shortened number, but no show with more than 1000

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package org.schabi.newpipe.extractor.services.youtube.extractors;
22

33
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl;
4-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
4+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow;
55
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem;
66
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
77

@@ -26,8 +26,7 @@ public YoutubeMixOrPlaylistInfoItemExtractor(final JsonObject mixInfoItem) {
2626

2727
@Override
2828
public String getName() throws ParsingException {
29-
return getTextFromObject(mixInfoItem.getObject("title"))
30-
.orElseThrow(() -> new ParsingException("Could not get name"));
29+
return getTextFromObjectOrThrow(mixInfoItem.getObject("title"), "name");
3130
}
3231

3332
@Override
@@ -66,10 +65,9 @@ public boolean isUploaderVerified() throws ParsingException {
6665

6766
@Override
6867
public long getStreamCount() throws ParsingException {
69-
final String countString = YoutubeParsingHelper.getTextFromObject(
70-
mixInfoItem.getObject("videoCountShortText"))
71-
.orElseThrow(() -> new ParsingException("Could not extract item count for"
72-
+ " playlist/mix info item"));
68+
final var textObject = mixInfoItem.getObject("videoCountShortText");
69+
final String countString = getTextFromObjectOrThrow(textObject,
70+
"item count for playlist/mix info item");
7371

7472
try {
7573
return Integer.parseInt(countString);

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,8 @@ public String getUploaderUrl() throws ParsingException {
8787
.getArray("commandRuns")
8888
.getObject(0)
8989
.getObject("onTap")
90-
.getObject("innertubeCommand"));
90+
.getObject("innertubeCommand"))
91+
.orElse(null);
9192
}
9293

9394
@Override
@@ -159,7 +160,7 @@ public String getUrl() throws ParsingException {
159160
return getUrlFromNavigationEndpoint(lockupViewModel.getObject("rendererContext")
160161
.getObject("commandContext")
161162
.getObject("onTap")
162-
.getObject("innertubeCommand"));
163+
.getObject("innertubeCommand")).orElse(null);
163164
}
164165

165166
@Nonnull

0 commit comments

Comments
 (0)