diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java index 91322acc..9cb5bc86 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java @@ -7,6 +7,7 @@ public final class ChannelTabs { public static final String LIVESTREAMS = "livestreams"; public static final String CHANNELS = "channels"; public static final String PLAYLISTS = "playlists"; + public static final String PODCASTS = "podcasts"; public static final String ALBUMS = "albums"; private ChannelTabs() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 7e917fbd..ecd0394e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -172,7 +172,16 @@ public KioskList getKioskList() throws ExtractionException { new YoutubeTrendingLinkHandlerFactory(), "Recommended Lives" ); - list.setDefaultKiosk("Recommended Lives"); + list.addKioskEntry( + (streamingService, url, id) -> new YoutubeTrendingExtractor( + YoutubeService.this, + new YoutubeTrendingLinkHandlerFactory().fromUrl(url), + id + ), + new YoutubeTrendingLinkHandlerFactory(), + "Recommended Podcasts" + ); + list.setDefaultKiosk("Recommended Podcasts"); } catch (final Exception e) { throw new ExtractionException(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index cd9b127f..ee5b285b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -674,8 +674,7 @@ private JsonObject getVideoTab() throws ParsingException { .getObject("commandMetadata").getObject("webCommandMetadata") .getString("url"); if (tabUrl != null) { - final String[] urlParts = tabUrl.split("/"); - final String urlSuffix = urlParts[urlParts.length - 1]; + final String urlSuffix = normalizeTabSuffix(tabUrl); switch (urlSuffix) { case "videos": @@ -685,6 +684,9 @@ private JsonObject getVideoTab() throws ParsingException { case "playlists": addTab.accept(ChannelTabs.PLAYLISTS); break; + case "podcasts": + addTab.accept(ChannelTabs.PODCASTS); + break; case "streams": addTab.accept(ChannelTabs.LIVESTREAMS); break; @@ -723,4 +725,29 @@ private JsonObject getVideoTab() throws ParsingException { return foundVideoTab; } + @Nonnull + private static String normalizeTabSuffix(@Nonnull final String tabUrl) { + String normalized = tabUrl; + final int queryIndex = normalized.indexOf('?'); + if (queryIndex >= 0) { + normalized = normalized.substring(0, queryIndex); + } + + final int fragmentIndex = normalized.indexOf('#'); + if (fragmentIndex >= 0) { + normalized = normalized.substring(0, fragmentIndex); + } + + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + + final int slashIndex = normalized.lastIndexOf('/'); + if (slashIndex < 0 || slashIndex == normalized.length() - 1) { + return normalized; + } + + return normalized.substring(slashIndex + 1); + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java index 2b05d3be..370c283c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java @@ -69,6 +69,8 @@ private String getChannelTabsParameters() throws ParsingException { return "EghyZWxlYXNlc_IGBQoDsgEA"; case ChannelTabs.PLAYLISTS: return "EglwbGF5bGlzdHPyBgQKAkIA"; + case ChannelTabs.PODCASTS: + return "Eghwb2RjYXN0c_IGBQoDugEA"; default: throw new ParsingException("Unsupported channel tab: " + name); } @@ -197,9 +199,10 @@ private JsonObject getTabData() throws ParsingException { JsonObject foundTab = null; for (final Object tab : tabs) { if (((JsonObject) tab).has("tabRenderer")) { - if (((JsonObject) tab).getObject("tabRenderer").getObject("endpoint") + final String tabUrl = ((JsonObject) tab).getObject("tabRenderer").getObject("endpoint") .getObject("commandMetadata").getObject("webCommandMetadata") - .getString("url").endsWith(urlSuffix)) { + .getString("url"); + if (tabUrl != null && normalizeTabUrl(tabUrl).endsWith(urlSuffix)) { foundTab = ((JsonObject) tab).getObject("tabRenderer"); break; } @@ -309,6 +312,11 @@ public String getUploaderName() { return channelIds.get(0); } }); + } else if (richItem.has("lockupViewModel")) { + commitLockupItemIfSupported(collector, + richItem.getObject("lockupViewModel"), channelIds); + } else { + return collectItem(collector, richItem, channelIds); } } else if (item.has("gridPlaylistRenderer")) { collector.commit(new YoutubePlaylistInfoItemExtractor( @@ -318,6 +326,12 @@ public String getUploaderName() { return channelIds.get(0); } }); + } else if (item.has("playlistRenderer")) { + collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor( + item.getObject("playlistRenderer"))); + } else if (item.has("radioRenderer")) { + collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor( + item.getObject("radioRenderer"))); } else if (item.has("gridChannelRenderer")) { collector.commit(new YoutubeChannelInfoItemExtractor( item.getObject("gridChannelRenderer"))); @@ -336,23 +350,73 @@ public String getUploaderName() { } else if (item.has("continuationItemRenderer")) { return item.getObject("continuationItemRenderer"); } else if (item.has("lockupViewModel")) { - final JsonObject lockupViewModel = item.getObject("lockupViewModel"); - final String contentType = lockupViewModel.getString("contentType"); - if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(contentType) - || "LOCKUP_CONTENT_TYPE_PODCAST".equals(contentType)) { - String channelName; - try { - channelName = getChannelName(); - } catch (Exception e) { - channelName = channelIds.get(0); - } - commitPlaylistLockup(collector, lockupViewModel, - channelName, null); - } + commitLockupItemIfSupported(collector, + item.getObject("lockupViewModel"), channelIds); } return null; } + private void commitLockupItemIfSupported(@Nonnull final MultiInfoItemsCollector collector, + @Nonnull final JsonObject lockupViewModel, + @Nonnull final List channelIds) { + final String contentType = lockupViewModel.getString("contentType"); + if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(contentType) + || "LOCKUP_CONTENT_TYPE_PODCAST".equals(contentType)) { + String channelName; + try { + channelName = getChannelName(); + } catch (final Exception e) { + channelName = channelIds.get(0); + } + commitPlaylistLockup(collector, lockupViewModel, channelName, null); + return; + } + + if ("LOCKUP_CONTENT_TYPE_VIDEO".equals(contentType) + || "LOCKUP_CONTENT_TYPE_EPISODE".equals(contentType)) { + collector.commit(new YoutubeLockupStreamInfoItemExtractor(lockupViewModel, + getTimeAgoParser()) { + @Override + public String getUploaderName() throws ParsingException { + try { + return super.getUploaderName(); + } catch (final ParsingException e) { + return channelIds.get(0); + } + } + + @Override + public String getUploaderUrl() throws ParsingException { + try { + return super.getUploaderUrl(); + } catch (final ParsingException e) { + return channelIds.get(1); + } + } + }); + } + } + + @Nonnull + private static String normalizeTabUrl(@Nonnull final String tabUrl) { + String normalized = tabUrl; + final int queryIndex = normalized.indexOf('?'); + if (queryIndex >= 0) { + normalized = normalized.substring(0, queryIndex); + } + + final int fragmentIndex = normalized.indexOf('#'); + if (fragmentIndex >= 0) { + normalized = normalized.substring(0, fragmentIndex); + } + + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + + return normalized; + } + @Nullable private Page getNextPageFrom(final JsonObject continuations, final List channelIds) throws IOException, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 529a167c..6d45b66f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -47,6 +47,12 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubeTrendingExtractor extends KioskExtractor { + private static final String KIOSK_RECOMMENDED_PODCASTS = "Recommended Podcasts"; + private static final String RECOMMENDED_LIVES_BROWSE_ID = "UC4R8DWoMoI7CAwX8_LjQHig"; + private static final String RECOMMENDED_PODCASTS_BROWSE_ID = "FEpodcasts_destination"; + private static final String RECOMMENDED_PODCASTS_PARAMS = "qgcCCAM="; + private static final long RECOMMENDED_PODCASTS_MAX_ITEMS = 40; + private JsonObject initialData; public YoutubeTrendingExtractor(final StreamingService service, @@ -58,13 +64,21 @@ public YoutubeTrendingExtractor(final StreamingService service, @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - // @formatter:off - final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), - getExtractorContentCountry()) - .value("browseId", "UC4R8DWoMoI7CAwX8_LjQHig") - .done()) - .getBytes(UTF_8); - // @formatter:on + final byte[] body; + if (KIOSK_RECOMMENDED_PODCASTS.equals(getId())) { + body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), + getExtractorContentCountry()) + .value("browseId", RECOMMENDED_PODCASTS_BROWSE_ID) + .value("params", RECOMMENDED_PODCASTS_PARAMS) + .done()) + .getBytes(UTF_8); + } else { + body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), + getExtractorContentCountry()) + .value("browseId", RECOMMENDED_LIVES_BROWSE_ID) + .done()) + .getBytes(UTF_8); + } initialData = getJsonPostResponse("browse", body, getExtractorLocalization()); } @@ -86,6 +100,8 @@ public String getName() throws ParsingException { } else if (header.has("carouselHeaderRenderer")) { name = header.getObject("carouselHeaderRenderer").getArray("contents").getObject(0) .getObject("topicChannelDetailsRenderer").getObject("title").getString("simpleText"); + } else if (header.has("pageHeaderRenderer")) { + name = header.getObject("pageHeaderRenderer").getString("pageTitle"); } if (isNullOrEmpty(name)) { @@ -99,6 +115,9 @@ public String getName() throws ParsingException { public InfoItemsPage getInitialPage() throws ParsingException { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final TimeAgoParser timeAgoParser = getTimeAgoParser(); + final long maximumItems = KIOSK_RECOMMENDED_PODCASTS.equals(getId()) + ? RECOMMENDED_PODCASTS_MAX_ITEMS + : Long.MAX_VALUE; final JsonObject tabContent = getTrendingTabRenderer().getObject("content"); if (tabContent.has("richGridRenderer")) { @@ -110,8 +129,10 @@ public InfoItemsPage getInitialPage() throws ParsingException { // Filter Trending shorts and Recently trending sections .filter(content -> content.has("richItemRenderer")) .map(content -> content.getObject("richItemRenderer") - .getObject("content") - .getObject("videoRenderer")) + .getObject("content")) + .filter(content -> content.has("videoRenderer")) + .map(content -> content.getObject("videoRenderer")) + .limit(maximumItems) .forEachOrdered(videoRenderer -> collector.commit( new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser))); } else if (tabContent.has("sectionListRenderer")) { @@ -136,6 +157,7 @@ public InfoItemsPage getInitialPage() throws ParsingException { .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) .map(item -> item.getObject("videoRenderer")) + .limit(maximumItems) .forEachOrdered(videoRenderer -> collector.commit( new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser))); } @@ -159,6 +181,7 @@ public InfoItemsPage getInitialPage() throws ParsingException { .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) .map(item -> item.getObject("gridVideoRenderer")) + .limit(maximumItems) .forEachOrdered(videoRenderer -> collector.commit( new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser))); } @@ -182,11 +205,12 @@ public InfoItemsPage getInitialPage() throws ParsingException { .map(content -> content.getObject("richItemRenderer") .getObject("content") .getObject("videoRenderer")) + .limit(maximumItems) .forEachOrdered(videoRenderer -> collector.commit( new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser))); } if (collector.getItems().isEmpty()) { - throw new ParsingException("Could not get trending page"); + throw new ParsingException("Could not get kiosk page: " + getId()); } if (ServiceList.YouTube.getFilterTypes().contains("recommendations")) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java index 36863fd1..ac9ee16a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java @@ -28,6 +28,8 @@ public static String getUrlSuffix(final String tab) throws ParsingException { return "/videos"; case ChannelTabs.PLAYLISTS: return "/playlists"; + case ChannelTabs.PODCASTS: + return "/podcasts"; case ChannelTabs.LIVESTREAMS: return "/streams"; case ChannelTabs.SHORTS: diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java index 3f625562..4b118b9f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java @@ -34,21 +34,40 @@ public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { + private static final String TRENDING_ID = "Trending"; + private static final String RECOMMENDED_LIVES_ID = "Recommended Lives"; + private static final String RECOMMENDED_PODCASTS_ID = "Recommended Podcasts"; + private static final String TRENDING_URL = "https://www.youtube.com/feed/trending"; + private static final String RECOMMENDED_LIVES_URL = + "https://www.youtube.com/channel/UC4R8DWoMoI7CAwX8_LjQHig"; + private static final String RECOMMENDED_PODCASTS_URL = "https://www.youtube.com/podcasts/videos"; + public String getUrl(final String id, final List contentFilters, final List sortFilter) { - if(!id.equals("Trending")){ - return "https://www.youtube.com/channel/UC4R8DWoMoI7CAwX8_LjQHig"; + if (TRENDING_ID.equals(id)) { + return TRENDING_URL; + } + + if (RECOMMENDED_PODCASTS_ID.equals(id)) { + return RECOMMENDED_PODCASTS_URL; } - return "https://www.youtube.com/feed/trending"; + + return RECOMMENDED_LIVES_URL; } @Override public String getId(final String url) { - if(url.equals("https://www.youtube.com/feed/trending")){ - return "Trending"; + if (TRENDING_URL.equals(url)) { + return TRENDING_ID; } - return "Recommended Lives"; + + if (RECOMMENDED_PODCASTS_URL.equals(url) + || "https://www.youtube.com/podcasts".equals(url)) { + return RECOMMENDED_PODCASTS_ID; + } + + return RECOMMENDED_LIVES_ID; } @Override @@ -62,6 +81,11 @@ public boolean onAcceptUrl(final String url) { final String urlPath = urlObj.getPath(); return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidiousURL(urlObj)) - && (urlPath.equals("/feed/trending") || urlPath.equals("/channel/UC4R8DWoMoI7CAwX8_LjQHig")); + && (urlPath.equals("/feed/trending") + || urlPath.equals("/channel/UC4R8DWoMoI7CAwX8_LjQHig") + || urlPath.equals("/podcasts") + || urlPath.equals("/podcasts/") + || urlPath.equals("/podcasts/videos") + || urlPath.equals("/podcasts/videos/")); } }