Skip to content

Commit a338e4e

Browse files
XiangRongLinXiangRongLin
committed
[Youtube] Apply review suggestions and avoid channel mix edge case
1 parent 22d2f7e commit a338e4e

4 files changed

Lines changed: 70 additions & 31 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,13 +214,44 @@ public static boolean isYoutubeMixId(final String playlistId) {
214214

215215
/**
216216
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
217-
* Ids from a YouTube Music Mix start with "RD"
217+
* Ids from a YouTube Music Mix start with "RDAMVM"
218218
* @param playlistId
219219
* @return Whether given id belongs to a YouTube Music Mix
220220
*/
221221
public static boolean isYoutubeMusicMixId(final String playlistId) {
222222
return playlistId.startsWith("RDAMVM");
223223
}
224+
/**
225+
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
226+
* Ids from a YouTube channel Mix start with "RDCM"
227+
* @return Whether given id belongs to a YouTube Channel Mix
228+
*/
229+
public static boolean isYoutubeChannelMixId(final String playlistId) {
230+
return playlistId.startsWith("RDCM");
231+
}
232+
233+
/**
234+
* Extracts the video id from the playlist id for Mixes.
235+
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
236+
*/
237+
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
238+
if (playlistId.startsWith("RDMM")) { //My Mix
239+
return playlistId.substring(4);
240+
241+
} else if (playlistId.startsWith("RDAMVM")) { //Music mix
242+
return playlistId.substring(6);
243+
244+
} else if (playlistId.startsWith("RMCM")) { //Channel mix
245+
//Channel mix are build with RMCM{channelId}, so videoId can't be determined
246+
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
247+
248+
} else if (playlistId.startsWith("RD")) { // Normal mix
249+
return playlistId.substring(2);
250+
251+
} else { //not a mix
252+
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
253+
}
254+
}
224255

225256
public static JsonObject getInitialData(String html) throws ParsingException {
226257
try {
@@ -362,7 +393,7 @@ public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, Re
362393
.end()
363394
.value("query", "test")
364395
.value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
365-
.end().done().getBytes(StandardCharsets.UTF_8);
396+
.end().done().getBytes("UTF-8");
366397
// @formatter:on
367398

368399
Map<String, List<String>> headers = new HashMap<>();

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
2929
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
3030
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray;
31+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
3132

3233
/**
3334
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
@@ -40,7 +41,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
4041
* YouTube identifies mixes based on this cookie. With this information it can generate
4142
* continuations without duplicates.
4243
*/
43-
private static final String COOKIE_NAME = "VISITOR_INFO1_LIVE";
44+
public static final String COOKIE_NAME = "VISITOR_INFO1_LIVE";
4445

4546
private JsonObject initialData;
4647
private JsonObject playlistData;
@@ -124,11 +125,7 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException
124125
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
125126
collectStreamsFrom(collector, playlistData.getArray("contents"));
126127
return new InfoItemsPage<>(collector,
127-
new Page(getNextPageUrl(), Collections.singletonMap(COOKIE_NAME, cookieValue)));
128-
}
129-
130-
private String getNextPageUrl() throws ExtractionException {
131-
return getNextPageUrlFrom(playlistData);
128+
new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue)));
132129
}
133130

134131
private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {
@@ -146,9 +143,11 @@ private String getNextPageUrlFrom(final JsonObject playlistJson) throws Extracti
146143
@Override
147144
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
148145
throws ExtractionException, IOException {
149-
if (page == null || page.getUrl().isEmpty()) {
150-
throw new ExtractionException(
151-
new IllegalArgumentException("Page url is empty or null"));
146+
if (page == null || isNullOrEmpty(page.getUrl())) {
147+
throw new IllegalArgumentException("Page url is empty or null");
148+
}
149+
if (!page.getCookies().containsKey(COOKIE_NAME)) {
150+
throw new IllegalArgumentException("Cooke '" + COOKIE_NAME + "' is missing");
152151
}
153152

154153
final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization());

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
22

3+
import java.net.MalformedURLException;
4+
import java.net.URL;
5+
import java.util.List;
36
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
47
import org.schabi.newpipe.extractor.exceptions.ParsingException;
58
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
@@ -8,10 +11,6 @@
811
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
912
import org.schabi.newpipe.extractor.utils.Utils;
1013

11-
import java.net.MalformedURLException;
12-
import java.net.URL;
13-
import java.util.List;
14-
1514
public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
1615

1716
private static final YoutubePlaylistLinkHandlerFactory INSTANCE =
@@ -58,6 +57,12 @@ public String getId(final String url) throws ParsingException {
5857
"YouTube Music Mix playlists are not yet supported");
5958
}
6059

60+
if (YoutubeParsingHelper.isYoutubeChannelMixId(listID)
61+
&& Utils.getQueryValue(urlObj, "v") == null) {
62+
//Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId
63+
throw new ContentNotSupportedException("Channel Mix without a video id are not supported");
64+
}
65+
6166
return listID;
6267
} catch (final Exception exception) {
6368
throw new ParsingException("Error could not parse url :" + exception.getMessage(),
@@ -89,7 +94,7 @@ public ListLinkHandler fromUrl(final String url) throws ParsingException {
8994
if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) {
9095
String videoID = Utils.getQueryValue(urlObj, "v");
9196
if (videoID == null) {
92-
videoID = listID.substring(2);
97+
videoID = YoutubeParsingHelper.extractVideoIdFromMixId(listID);
9398
}
9499
final String newUrl = "https://www.youtube.com/watch?v=" + videoID
95100
+ "&list=" + listID;

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,8 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3-
import static org.hamcrest.CoreMatchers.containsString;
4-
import static org.hamcrest.CoreMatchers.startsWith;
5-
import static org.hamcrest.MatcherAssert.assertThat;
6-
import static org.junit.Assert.assertEquals;
7-
import static org.junit.Assert.assertFalse;
8-
import static org.junit.Assert.assertTrue;
9-
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
10-
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
11-
3+
import java.util.Collections;
124
import java.util.HashSet;
5+
import java.util.Map;
136
import java.util.Set;
147
import org.hamcrest.MatcherAssert;
158
import org.junit.BeforeClass;
@@ -31,6 +24,15 @@
3124
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
3225
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
3326

27+
import static org.hamcrest.CoreMatchers.containsString;
28+
import static org.hamcrest.CoreMatchers.startsWith;
29+
import static org.hamcrest.MatcherAssert.assertThat;
30+
import static org.junit.Assert.assertEquals;
31+
import static org.junit.Assert.assertFalse;
32+
import static org.junit.Assert.assertTrue;
33+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
34+
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
35+
3436
@RunWith(Suite.class)
3537
@SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class})
3638
public class YoutubeMixPlaylistExtractorTest {
@@ -41,6 +43,8 @@ public class YoutubeMixPlaylistExtractorTest {
4143
"Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
4244

4345
private static YoutubeMixPlaylistExtractor extractor;
46+
private static Map<String, String> dummyCookie
47+
= Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
4448

4549
public static class Mix {
4650

@@ -83,8 +87,8 @@ public void getInitialPage() throws Exception {
8387
@Test
8488
public void getPage() throws Exception {
8589
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
86-
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID
87-
+ PBJ));
90+
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID
91+
+ PBJ, dummyCookie));
8892
assertFalse(streams.getItems().isEmpty());
8993
assertTrue(streams.hasNextPage());
9094
}
@@ -157,7 +161,7 @@ public void getInitialPage() throws Exception {
157161
public void getPage() throws Exception {
158162
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
159163
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
160-
+ VIDEO_ID + INDEX + PBJ));
164+
+ VIDEO_ID + INDEX + PBJ, dummyCookie));
161165
assertFalse(streams.getItems().isEmpty());
162166
assertTrue(streams.hasNextPage());
163167
}
@@ -229,7 +233,7 @@ public void getInitialPage() throws Exception {
229233
public void getPage() throws Exception {
230234
final InfoItemsPage<StreamInfoItem> streams =
231235
extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
232-
+ "&list=RDMM" + VIDEO_ID + PBJ));
236+
+ "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie));
233237
assertFalse(streams.getItems().isEmpty());
234238
assertTrue(streams.hasNextPage());
235239
}
@@ -267,7 +271,7 @@ public static void setUp() {
267271
NewPipe.init(DownloaderTestImpl.getInstance());
268272
}
269273

270-
@Test(expected = ExtractionException.class)
274+
@Test(expected = IllegalArgumentException.class)
271275
public void getPageEmptyUrl() throws Exception {
272276
extractor = (YoutubeMixPlaylistExtractor) YouTube
273277
.getPlaylistExtractor(
@@ -328,7 +332,7 @@ public void getInitialPage() throws Exception {
328332
public void getPage() throws Exception {
329333
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
330334
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
331-
+ "&list=RDCM" + CHANNEL_ID + PBJ));
335+
+ "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie));
332336
assertFalse(streams.getItems().isEmpty());
333337
assertTrue(streams.hasNextPage());
334338
}

0 commit comments

Comments
 (0)