Skip to content

Commit df38b19

Browse files
XiangRongLinXiangRongLin
committed
[Youtube] Add tests and take thumbnail image always from first video of mix
Also fix getThumbnailUrl for "My Mix"
1 parent 68a3948 commit df38b19

4 files changed

Lines changed: 514 additions & 129 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 148 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -1,163 +1,182 @@
11
package org.schabi.newpipe.extractor.services.youtube.extractors;
22

3-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
43
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
54

65
import com.grack.nanojson.JsonArray;
76
import com.grack.nanojson.JsonObject;
87
import java.io.IOException;
98
import javax.annotation.Nonnull;
109
import javax.annotation.Nullable;
11-
import org.jsoup.nodes.Document;
12-
import org.jsoup.nodes.Element;
1310
import org.schabi.newpipe.extractor.StreamingService;
1411
import org.schabi.newpipe.extractor.downloader.Downloader;
15-
import org.schabi.newpipe.extractor.downloader.Response;
1612
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
1713
import org.schabi.newpipe.extractor.exceptions.ParsingException;
18-
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
1914
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
2015
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
2116
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
22-
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
2317
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
2418
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
2519

2620
/**
27-
* A YoutubePlaylistExtractor for a mix (auto-generated playlist).
28-
* It handles urls in the format of "youtube.com/watch?v=videoId&list=playlistId"
21+
* A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of
22+
* "youtube.com/watch?v=videoId&list=playlistId"
2923
*/
3024
public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
3125

32-
private JsonObject initialData;
33-
private JsonObject playlistData;
34-
35-
public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
36-
super(service, linkHandler);
37-
}
38-
39-
@Override
40-
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
41-
final String url = getUrl() + "&pbj=1";
42-
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
43-
initialData = ajaxJson.getObject(3).getObject("response");
44-
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist");
45-
System.out.println();
46-
}
47-
48-
@Nonnull
49-
@Override
50-
public String getName() throws ParsingException {
51-
try {
52-
final String name = playlistData.getString("title");
53-
if (name!= null) return name;
54-
else return "";
55-
} catch (Exception e) {
56-
throw new ParsingException("Could not get playlist name", e);
26+
private final static String CONTENTS = "contents";
27+
private final static String RESPONSE = "response";
28+
private final static String PLAYLIST = "playlist";
29+
private final static String TWO_COLUMN_WATCH_NEXT_RESULTS = "twoColumnWatchNextResults";
30+
private final static String PLAYLIST_PANEL_VIDEO_RENDERER = "playlistPanelVideoRenderer";
31+
32+
private JsonObject playlistData;
33+
34+
public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
35+
super(service, linkHandler);
36+
}
37+
38+
@Override
39+
public void onFetchPage(@Nonnull Downloader downloader)
40+
throws IOException, ExtractionException {
41+
final String url = getUrl() + "&pbj=1";
42+
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
43+
JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE);
44+
try {
45+
playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS)
46+
.getObject(PLAYLIST).getObject(PLAYLIST);
47+
} catch (NullPointerException e) {
48+
throw new ExtractionException(e);
49+
}
50+
51+
}
52+
53+
@Nonnull
54+
@Override
55+
public String getName() throws ParsingException {
56+
try {
57+
final String name = playlistData.getString("title");
58+
if (name != null) {
59+
return name;
60+
} else {
61+
return "";
62+
}
63+
} catch (Exception e) {
64+
throw new ParsingException("Could not get playlist name", e);
65+
}
66+
}
67+
68+
@Override
69+
public String getThumbnailUrl() throws ParsingException {
70+
try {
71+
final String playlistId = playlistData.getString("playlistId");
72+
final String videoId;
73+
if (playlistId.startsWith("RDMM")) {
74+
videoId = playlistId.substring(4);
75+
} else {
76+
videoId = playlistId.substring(2);
77+
}
78+
if (videoId.isEmpty()) {
79+
throw new ParsingException("");
80+
}
81+
return getThumbnailUrlFromId(videoId);
82+
} catch (Exception e) {
83+
throw new ParsingException("Could not get playlist thumbnail", e);
84+
}
5785
}
58-
}
59-
60-
@Override
61-
public String getThumbnailUrl() throws ParsingException {
62-
try {
63-
final String videoId = playlistData.getArray("contents").getObject(0)
64-
.getObject("playlistPanelVideoRenderer").getString("videoId");
65-
if (videoId == null || videoId.isEmpty()) throw new ParsingException("");
66-
return getThumbnailUrlFromId(videoId);
67-
} catch (Exception e) {
68-
throw new ParsingException("Could not get playlist thumbnail", e);
86+
87+
@Override
88+
public String getBannerUrl() {
89+
return "";
6990
}
70-
}
71-
72-
@Override
73-
public String getBannerUrl() {
74-
return "";
75-
}
76-
77-
@Override
78-
public String getUploaderUrl() {
79-
//Youtube mix are auto-generated
80-
return "";
81-
}
82-
83-
@Override
84-
public String getUploaderName() {
85-
//Youtube mix are auto-generated
86-
return "";
87-
}
88-
89-
@Override
90-
public String getUploaderAvatarUrl() {
91-
//Youtube mix are auto-generated
92-
return "";
93-
}
94-
95-
@Override
96-
public long getStreamCount() {
97-
// Auto-generated playlist always start with 25 videos and are endless
98-
return 25;
99-
}
100-
101-
@Nonnull
102-
@Override
103-
public InfoItemsPage<StreamInfoItem> getInitialPage() {
104-
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
105-
collectStreamsFrom(collector, playlistData.getArray("contents"));
106-
return new InfoItemsPage<>(collector, getNextPageUrl());
107-
}
108-
109-
@Override
110-
public String getNextPageUrl() {
111-
final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents")
112-
.get(playlistData.getArray("contents").size() - 1));
113-
final String lastStreamId = lastStream.getObject("playlistPanelVideoRenderer")
114-
.getString("videoId");
115-
return "https://youtube.com" + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")
116-
.getObject("commandMetadata").getObject("webCommandMetadata").getString("url") + "&pbj=1";
117-
}
118-
119-
@Override
120-
public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl)
121-
throws ExtractionException, IOException {
122-
if (pageUrl == null || pageUrl.isEmpty()) {
123-
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
91+
92+
@Override
93+
public String getUploaderUrl() {
94+
//Youtube mix are auto-generated
95+
return "";
12496
}
12597

126-
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
127-
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
128-
playlistData =
129-
ajaxJson.getObject(3).getObject("response").getObject("contents")
130-
.getObject("twoColumnWatchNextResults").getObject("playlist")
131-
.getObject("playlist");
132-
final JsonArray streams = playlistData.getArray("contents");
133-
//Because continuation requests are created with the last video of previous request as start
134-
streams.remove(0);
135-
collectStreamsFrom(collector, streams);
136-
return new InfoItemsPage<>(collector, getNextPageUrl());
137-
}
138-
139-
private void collectStreamsFrom(
140-
@Nonnull StreamInfoItemsCollector collector,
141-
@Nullable JsonArray streams) {
142-
collector.reset();
143-
144-
if (streams == null) {
145-
return;
98+
@Override
99+
public String getUploaderName() {
100+
//Youtube mix are auto-generated
101+
return "";
146102
}
147103

148-
final TimeAgoParser timeAgoParser = getTimeAgoParser();
104+
@Override
105+
public String getUploaderAvatarUrl() {
106+
//Youtube mix are auto-generated
107+
return "";
108+
}
149109

150-
for (Object stream : streams) {
151-
if (stream instanceof JsonObject) {
152-
JsonObject streamInfo = ((JsonObject) stream).getObject("playlistPanelVideoRenderer");
153-
if (streamInfo != null) {
154-
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
110+
@Override
111+
public long getStreamCount() {
112+
// Auto-generated playlist always start with 25 videos and are endless
113+
return 25;
114+
}
115+
116+
@Nonnull
117+
@Override
118+
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
119+
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
120+
collectStreamsFrom(collector, playlistData.getArray(CONTENTS));
121+
return new InfoItemsPage<>(collector, getNextPageUrl());
122+
}
123+
124+
@Override
125+
public String getNextPageUrl() throws ExtractionException {
126+
final JsonObject lastStream = ((JsonObject) playlistData.getArray(CONTENTS)
127+
.get(playlistData.getArray(CONTENTS).size() - 1));
128+
if (lastStream == null || lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) == null) {
129+
throw new ExtractionException("Could not extract next page url");
130+
}
131+
return "https://youtube.com" + lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER)
132+
.getObject("navigationEndpoint").getObject("commandMetadata")
133+
.getObject("webCommandMetadata").getString("url") + "&pbj=1";
134+
}
135+
136+
@Override
137+
public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl)
138+
throws ExtractionException, IOException {
139+
if (pageUrl == null || pageUrl.isEmpty()) {
140+
throw new ExtractionException(
141+
new IllegalArgumentException("Page url is empty or null"));
155142
}
156-
}
143+
144+
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
145+
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
146+
playlistData =
147+
ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS)
148+
.getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS).getObject(PLAYLIST)
149+
.getObject(PLAYLIST);
150+
final JsonArray streams = playlistData.getArray(CONTENTS);
151+
//Because continuation requests are created with the last video of previous request as start
152+
streams.remove(0);
153+
collectStreamsFrom(collector, streams);
154+
return new InfoItemsPage<>(collector, getNextPageUrl());
157155
}
158-
}
159156

160-
private String getThumbnailUrlFromId(String videoId) {
161-
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
162-
}
157+
private void collectStreamsFrom(
158+
@Nonnull StreamInfoItemsCollector collector,
159+
@Nullable JsonArray streams) {
160+
collector.reset();
161+
162+
if (streams == null) {
163+
return;
164+
}
165+
166+
final TimeAgoParser timeAgoParser = getTimeAgoParser();
167+
168+
for (Object stream : streams) {
169+
if (stream instanceof JsonObject) {
170+
JsonObject streamInfo = ((JsonObject) stream)
171+
.getObject(PLAYLIST_PANEL_VIDEO_RENDERER);
172+
if (streamInfo != null) {
173+
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
174+
}
175+
}
176+
}
177+
}
178+
179+
private String getThumbnailUrlFromId(String videoId) {
180+
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
181+
}
163182
}

0 commit comments

Comments
 (0)