Skip to content

Commit 3ff8619

Browse files
XiangRongLinXiangRongLin
committed
[Youtube] apply wb9688 suggestion (mix)
Channel mix adjusments and test Don't accept youtube music mix urls as playlist Don't override playlistData to keep getInitialPage() Remove json constants Indentation
1 parent 822cf30 commit 3ff8619

5 files changed

Lines changed: 64 additions & 52 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -193,13 +193,23 @@ public static OffsetDateTime parseDateFrom(String textualUploadDate) throws Pars
193193
}
194194

195195
/**
196-
* Checks if the given playlist id is a mix (auto-generated playlist)
197-
* Ids from a mix start with "RD"
196+
* Checks if the given playlist id is a youtube mix (auto-generated playlist)
197+
* Ids from a youtube mix start with "RD"
198198
* @param playlistId
199-
* @return Whether given id belongs to a mix
199+
* @return Whether given id belongs to a youtube mix
200200
*/
201201
public static boolean isYoutubeMixId(String playlistId) {
202-
return playlistId.startsWith("RD");
202+
return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId);
203+
}
204+
205+
/**
206+
* Checks if the given playlist id is a youtube music mix (auto-generated playlist)
207+
* Ids from a youtube music mix start with "RD"
208+
* @param playlistId
209+
* @return Whether given id belongs to a youtube music mix
210+
*/
211+
public static boolean isYoutubeMusicMixId(String playlistId) {
212+
return playlistId.startsWith("RDAMVM");
203213
}
204214

205215
public static JsonObject getInitialData(String html) throws ParsingException {
@@ -427,9 +437,9 @@ public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint)
427437
StringBuilder url = new StringBuilder();
428438
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
429439
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
430-
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
440+
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
431441
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
432-
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
442+
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
433443
return url.toString();
434444
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
435445
return "https://www.youtube.com/playlist?list=" +
@@ -457,6 +467,7 @@ public static String getTextFromObject(JsonObject textObject, boolean html) thro
457467
if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
458468
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
459469
if (!isNullOrEmpty(url)) {
470+
url = url.replaceAll("&", "&");
460471
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
461472
continue;
462473
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ public KioskList getKioskList() throws ExtractionException {
144144
public KioskExtractor createNewKiosk(StreamingService streamingService,
145145
String url,
146146
String id)
147-
throws ExtractionException {
147+
throws ExtractionException {
148148
return new YoutubeTrendingExtractor(YoutubeService.this,
149149
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id);
150150
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 40 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.schabi.newpipe.extractor.services.youtube.extractors;
22

33
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
4+
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
45

56
import com.grack.nanojson.JsonArray;
67
import com.grack.nanojson.JsonObject;
@@ -25,13 +26,7 @@
2526
*/
2627
public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
2728

28-
29-
private final static String CONTENTS = "contents";
30-
private final static String RESPONSE = "response";
31-
private final static String PLAYLIST = "playlist";
32-
private final static String TWO_COLUMN_WATCH_NEXT_RESULTS = "twoColumnWatchNextResults";
33-
private final static String PLAYLIST_PANEL_VIDEO_RENDERER = "playlistPanelVideoRenderer";
34-
29+
private JsonObject initialData;
3530
private JsonObject playlistData;
3631

3732
public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
@@ -43,9 +38,9 @@ public void onFetchPage(@Nonnull Downloader downloader)
4338
throws IOException, ExtractionException {
4439
final String url = getUrl() + "&pbj=1";
4540
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
46-
JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE);
47-
playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMN_WATCH_NEXT_RESULTS)
48-
.getObject(PLAYLIST).getObject(PLAYLIST);
41+
initialData = ajaxJson.getObject(3).getObject("response");
42+
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
43+
.getObject("playlist").getObject("playlist");
4944
}
5045

5146
@Nonnull
@@ -62,7 +57,14 @@ public String getName() throws ParsingException {
6257
public String getThumbnailUrl() throws ParsingException {
6358
try {
6459
final String playlistId = playlistData.getString("playlistId");
65-
return getThumbnailUrlFromId(playlistId);
60+
try {
61+
return getThumbnailUrlFromPlaylistId(playlistId);
62+
} catch (ParsingException e) {
63+
//fallback to thumbnail of current video. Always the case for channel mix
64+
return getThumbnailUrlFromVideoId(
65+
initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint")
66+
.getString("videoId"));
67+
}
6668
} catch (Exception e) {
6769
throw new ParsingException("Could not get playlist thumbnail", e);
6870
}
@@ -101,20 +103,26 @@ public long getStreamCount() {
101103
@Override
102104
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
103105
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
104-
collectStreamsFrom(collector, playlistData.getArray(CONTENTS));
106+
collectStreamsFrom(collector, playlistData.getArray("contents"));
105107
return new InfoItemsPage<>(collector, getNextPageUrl());
106108
}
107109

108110
@Override
109111
public String getNextPageUrl() throws ExtractionException {
110-
final JsonObject lastStream = ((JsonObject) playlistData.getArray(CONTENTS)
111-
.get(playlistData.getArray(CONTENTS).size() - 1));
112-
if (lastStream == null || lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) == null) {
112+
return getNextPageUrlFrom(playlistData);
113+
}
114+
115+
private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException {
116+
final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents")
117+
.get(playlistData.getArray("contents").size() - 1));
118+
if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) {
113119
throw new ExtractionException("Could not extract next page url");
114120
}
115-
return "https://youtube.com" + lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER)
116-
.getObject("navigationEndpoint").getObject("commandMetadata")
117-
.getObject("webCommandMetadata").getString("url") + "&pbj=1";
121+
//Index of video in mix is missing, but adding it doesn't appear to have any effect.
122+
//And since the index needs to be tracked by us, it is left out
123+
return getUrlFromNavigationEndpoint(
124+
lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint"))
125+
+ "&pbj=1";
118126
}
119127

120128
@Override
@@ -127,21 +135,20 @@ public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl)
127135

128136
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
129137
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
130-
playlistData =
131-
ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS)
132-
.getObject(TWO_COLUMN_WATCH_NEXT_RESULTS).getObject(PLAYLIST)
133-
.getObject(PLAYLIST);
134-
final JsonArray streams = playlistData.getArray(CONTENTS);
138+
JsonObject playlistData =
139+
ajaxJson.getObject(3).getObject("response").getObject("contents")
140+
.getObject("twoColumnWatchNextResults").getObject("playlist")
141+
.getObject("playlist");
142+
final JsonArray streams = playlistData.getArray("contents");
135143
//Because continuation requests are created with the last video of previous request as start
136144
streams.remove(0);
137145
collectStreamsFrom(collector, streams);
138-
return new InfoItemsPage<>(collector, getNextPageUrl());
146+
return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData));
139147
}
140148

141149
private void collectStreamsFrom(
142150
@Nonnull StreamInfoItemsCollector collector,
143151
@Nullable JsonArray streams) {
144-
collector.reset();
145152

146153
if (streams == null) {
147154
return;
@@ -152,24 +159,30 @@ private void collectStreamsFrom(
152159
for (Object stream : streams) {
153160
if (stream instanceof JsonObject) {
154161
JsonObject streamInfo = ((JsonObject) stream)
155-
.getObject(PLAYLIST_PANEL_VIDEO_RENDERER);
162+
.getObject("playlistPanelVideoRenderer");
156163
if (streamInfo != null) {
157164
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
158165
}
159166
}
160167
}
161168
}
162169

163-
private String getThumbnailUrlFromId(String playlistId) throws ParsingException {
170+
private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException {
164171
final String videoId;
165172
if (playlistId.startsWith("RDMM")) {
166173
videoId = playlistId.substring(4);
174+
} else if (playlistId.startsWith("RDCMUC")) {
175+
throw new ParsingException("is channel mix");
167176
} else {
168177
videoId = playlistId.substring(2);
169178
}
170179
if (videoId.isEmpty()) {
171180
throw new ParsingException("videoId is empty");
172181
}
182+
return getThumbnailUrlFromVideoId(videoId);
183+
}
184+
185+
private String getThumbnailUrlFromVideoId(String videoId) {
173186
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
174187
}
175188
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,12 @@ public String getId(String url) throws ParsingException {
6464
@Override
6565
public boolean onAcceptUrl(final String url) {
6666
try {
67-
getId(url);
67+
String playlistId = getId(url);
68+
//Because youtube music mix are not supported yet.
69+
return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId);
6870
} catch (ParsingException e) {
6971
return false;
7072
}
71-
return true;
7273
}
7374

7475
/**

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java‎

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
1414
import org.schabi.newpipe.extractor.NewPipe;
1515
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
16+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1617
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
1718
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1819

@@ -93,11 +94,6 @@ public void getPageMultipleTimes() throws Exception {
9394
public void getStreamCount() throws Exception {
9495
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
9596
}
96-
97-
@Test
98-
public void getStreamCount() throws Exception {
99-
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
100-
}
10197
}
10298

10399
public static class MixWithIndex {
@@ -166,11 +162,6 @@ public void getPageMultipleTimes() throws Exception {
166162
assertFalse(streams.getItems().isEmpty());
167163
}
168164

169-
@Test
170-
public void getStreamCount() {
171-
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
172-
}
173-
174165
@Test
175166
public void getStreamCount() throws Exception {
176167
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
@@ -264,12 +255,13 @@ public void getPageEmptyUrl() throws Exception {
264255
extractor.getPage("");
265256
}
266257

267-
@Test(expected = NullPointerException.class)
258+
@Test(expected = ExtractionException.class)
268259
public void invalidVideoId() throws Exception {
269260
extractor = (YoutubeMixPlaylistExtractor) YouTube
270261
.getPlaylistExtractor(
271262
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
272263
extractor.fetchPage();
264+
extractor.getName();
273265
}
274266
}
275267

@@ -329,10 +321,5 @@ public void getPage() throws Exception {
329321
public void getStreamCount() throws Exception {
330322
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
331323
}
332-
333-
@Test
334-
public void getStreamCount() throws Exception {
335-
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
336-
}
337324
}
338325
}

0 commit comments

Comments
 (0)