Skip to content

Commit 0efb854

Browse files
XiangRongLinXiangRongLin
committed
[Youtube] Implement mix extractor for auto-generated playlists.
-New YoutubeMixPlaylistExtractor, that extracts from a mix (auto-generated playlist). -The url has the format of "youtube.com/watch?v=videoID&playlistID", where playlistID always starts with "RD" and usually followed by the videoID. -Change YoutubePlaylistLinkHandlerFactory to create a linkhandler with the given url if it is a mix. -Change YoutubeService to return YoutubeMixPlaylistExtractor if the url is a mix.
1 parent 8ade913 commit 0efb854

4 files changed

Lines changed: 227 additions & 2 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,10 @@ public static OffsetDateTime parseDateFrom(String textualUploadDate) throws Pars
192192
}
193193
}
194194

195+
public static boolean isYoutubeMixId(String playlistId) {
196+
return playlistId.startsWith("RD");
197+
}
198+
195199
public static JsonObject getInitialData(String html) throws ParsingException {
196200
try {
197201
try {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,11 @@ public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
110110

111111
@Override
112112
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
113-
return new YoutubePlaylistExtractor(this, linkHandler);
113+
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
114+
return new YoutubeMixPlaylistExtractor(this, linkHandler);
115+
} else {
116+
return new YoutubePlaylistExtractor(this, linkHandler);
117+
}
114118
}
115119

116120
@Override
@@ -140,7 +144,7 @@ public KioskList getKioskList() throws ExtractionException {
140144
public KioskExtractor createNewKiosk(StreamingService streamingService,
141145
String url,
142146
String id)
143-
throws ExtractionException {
147+
throws ExtractionException {
144148
return new YoutubeTrendingExtractor(YoutubeService.this,
145149
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id);
146150
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
package org.schabi.newpipe.extractor.services.youtube.extractors;
2+
3+
import com.grack.nanojson.JsonObject;
4+
import com.grack.nanojson.JsonParser;
5+
import com.grack.nanojson.JsonParserException;
6+
import java.io.IOException;
7+
import javax.annotation.Nonnull;
8+
import javax.annotation.Nullable;
9+
import org.jsoup.Jsoup;
10+
import org.jsoup.nodes.Document;
11+
import org.jsoup.nodes.Element;
12+
import org.schabi.newpipe.extractor.StreamingService;
13+
import org.schabi.newpipe.extractor.downloader.Downloader;
14+
import org.schabi.newpipe.extractor.downloader.Response;
15+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
16+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
17+
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
18+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
19+
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
20+
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
21+
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
22+
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
23+
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
24+
25+
public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
26+
27+
private Document doc;
28+
29+
public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
30+
super(service, linkHandler);
31+
}
32+
33+
@Override
34+
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
35+
final String url = getUrl();
36+
final Response response = downloader.get(url, getExtractorLocalization());
37+
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
38+
}
39+
40+
@Nonnull
41+
@Override
42+
public String getName() throws ParsingException {
43+
try {
44+
return doc.select("div[class=\"playlist-info\"] h3[class=\"playlist-title\"]").first().text();
45+
} catch (Exception e) {
46+
throw new ParsingException("Could not get playlist name", e);
47+
}
48+
}
49+
50+
@Override
51+
public String getThumbnailUrl() throws ParsingException {
52+
try {
53+
return doc.select("ol[class*=\"playlist-videos-list\"] li").first().attr("data-thumbnail-url");
54+
} catch (Exception e) {
55+
throw new ParsingException("Could not get playlist thumbnail", e);
56+
}
57+
}
58+
59+
@Override
60+
public String getBannerUrl() {
61+
return "";
62+
}
63+
64+
@Override
65+
public String getUploaderUrl() {
66+
//Youtube mix are auto-generated
67+
return "";
68+
}
69+
70+
@Override
71+
public String getUploaderName() {
72+
//Youtube mix are auto-generated
73+
return "";
74+
}
75+
76+
@Override
77+
public String getUploaderAvatarUrl() {
78+
//Youtube mix are auto-generated
79+
return "";
80+
}
81+
82+
@Override
83+
public long getStreamCount() {
84+
// Auto-generated playlist always start with 25 videos and are endless
85+
// But the html doesn't have a continuation url
86+
return doc.select("ol[class*=\"playlist-videos-list\"] li").size();
87+
}
88+
89+
@Nonnull
90+
@Override
91+
public InfoItemsPage<StreamInfoItem> getInitialPage() {
92+
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
93+
Element ol = doc.select("ol[class*=\"playlist-videos-list\"]").first();
94+
collectStreamsFrom(collector, ol);
95+
return new InfoItemsPage<>(collector, getNextPageUrl());
96+
}
97+
98+
@Override
99+
public String getNextPageUrl() {
100+
return "";
101+
}
102+
103+
@Override
104+
public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) {
105+
//Continuations are not implemented
106+
return null;
107+
}
108+
109+
private void collectStreamsFrom(
110+
@Nonnull StreamInfoItemsCollector collector,
111+
@Nullable Element element) {
112+
collector.reset();
113+
114+
if (element == null) {
115+
return;
116+
}
117+
118+
final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory();
119+
final TimeAgoParser timeAgoParser = getTimeAgoParser();
120+
121+
for (final Element li : element.children()) {
122+
123+
collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
124+
125+
@Override
126+
public boolean isAd() {
127+
return false;
128+
}
129+
130+
@Override
131+
public String getUrl() throws ParsingException {
132+
try {
133+
return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl();
134+
} catch (Exception e) {
135+
throw new ParsingException("Could not get web page url for the video", e);
136+
}
137+
}
138+
139+
@Override
140+
public String getName() throws ParsingException {
141+
try {
142+
return li.attr("data-video-title");
143+
} catch (Exception e) {
144+
throw new ParsingException("Could not get name", e);
145+
}
146+
}
147+
148+
@Override
149+
public long getDuration() throws ParsingException {
150+
//Not present in doc
151+
return 0;
152+
}
153+
154+
@Override
155+
public String getUploaderName() throws ParsingException {
156+
try {
157+
return li.select(
158+
"div[class=\"playlist-video-description\"]"
159+
+ "span[class=\"video-uploader-byline\"]")
160+
.first()
161+
.text();
162+
} catch (Exception e) {
163+
throw new ParsingException("Could not get uploader", e);
164+
}
165+
}
166+
167+
@Override
168+
public String getUploaderUrl() {
169+
//Not present in doc
170+
return "";
171+
}
172+
173+
@Override
174+
public String getTextualUploadDate() {
175+
//Not present in doc
176+
return "";
177+
}
178+
179+
@Override
180+
public long getViewCount() {
181+
return -1;
182+
}
183+
184+
@Override
185+
public String getThumbnailUrl() throws ParsingException {
186+
try {
187+
return "https://i.ytimg.com/vi/" + streamLinkHandlerFactory.fromUrl(getUrl()).getId()
188+
+ "/hqdefault.jpg";
189+
} catch (Exception e) {
190+
throw new ParsingException("Could not get thumbnail url", e);
191+
}
192+
}
193+
});
194+
}
195+
}
196+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@
22

33
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
44
import org.schabi.newpipe.extractor.exceptions.ParsingException;
5+
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
6+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
57
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
68
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
79
import org.schabi.newpipe.extractor.utils.Utils;
810

11+
import java.net.MalformedURLException;
912
import java.net.URL;
1013
import java.util.List;
1114

@@ -67,4 +70,22 @@ public boolean onAcceptUrl(final String url) {
6770
}
6871
return true;
6972
}
73+
74+
@Override
75+
public ListLinkHandler fromUrl(String url) throws ParsingException {
76+
try {
77+
URL urlObj = Utils.stringToURL(url);
78+
String listID = Utils.getQueryValue(urlObj, "list");
79+
if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) {
80+
String videoID = Utils.getQueryValue(urlObj, "v");
81+
String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID;
82+
return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url),
83+
getSortFilter(url));
84+
}
85+
} catch (MalformedURLException exception) {
86+
throw new ParsingException("Error could not parse url :" + exception.getMessage(),
87+
exception);
88+
}
89+
return super.fromUrl(url);
90+
}
7091
}

0 commit comments

Comments
 (0)