Skip to content

Commit 85fa006

Browse files
StypoxStypox
authored
Merge pull request #280 from XiangRongLin/mixPL
Extractor for youtube mix (auto-generated playlist)
2 parents 2b622fd + f90f6fc commit 85fa006

10 files changed

Lines changed: 818 additions & 29 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems;
22

33
import com.grack.nanojson.JsonObject;
4+
5+
import org.schabi.newpipe.extractor.ListExtractor;
46
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
57
import org.schabi.newpipe.extractor.exceptions.ParsingException;
68

@@ -23,7 +25,7 @@ public long getSubscriberCount() {
2325

2426
@Override
2527
public long getStreamCount() {
26-
return -1;
28+
return ListExtractor.ITEM_COUNT_UNKNOWN;
2729
}
2830

2931
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 116 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@
55
import com.grack.nanojson.JsonParser;
66
import com.grack.nanojson.JsonParserException;
77
import com.grack.nanojson.JsonWriter;
8+
89
import org.jsoup.Jsoup;
910
import org.jsoup.nodes.Document;
11+
import org.schabi.newpipe.extractor.Page;
1012
import org.schabi.newpipe.extractor.downloader.Response;
1113
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
1214
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -21,6 +23,7 @@
2123
import java.net.MalformedURLException;
2224
import java.net.URL;
2325
import java.net.URLDecoder;
26+
import java.nio.charset.StandardCharsets;
2427
import java.time.LocalDate;
2528
import java.time.OffsetDateTime;
2629
import java.time.ZoneOffset;
@@ -35,6 +38,7 @@
3538
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
3639
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
3740
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
41+
import static org.schabi.newpipe.extractor.utils.Utils.join;
3842

3943
/*
4044
* Created by Christian Schabesberger on 02.03.16.
@@ -61,6 +65,12 @@ public class YoutubeParsingHelper {
6165
private YoutubeParsingHelper() {
6266
}
6367

68+
/**
69+
* The official youtube app supports intents in this format, where after the ':' is the videoId.
70+
* Accordingly there are other apps sharing streams in this format.
71+
*/
72+
public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube";
73+
6474
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
6575
private static String clientVersion;
6676

@@ -192,6 +202,57 @@ public static OffsetDateTime parseDateFrom(String textualUploadDate) throws Pars
192202
}
193203
}
194204

205+
/**
206+
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
207+
* Ids from a YouTube Mix start with "RD"
208+
* @param playlistId
209+
* @return Whether given id belongs to a YouTube Mix
210+
*/
211+
public static boolean isYoutubeMixId(final String playlistId) {
212+
return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId);
213+
}
214+
215+
/**
216+
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
217+
* Ids from a YouTube Music Mix start with "RDAMVM"
218+
* @param playlistId
219+
* @return Whether given id belongs to a YouTube Music Mix
220+
*/
221+
public static boolean isYoutubeMusicMixId(final String playlistId) {
222+
return playlistId.startsWith("RDAMVM");
223+
}
224+
/**
225+
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
226+
* Ids from a YouTube channel Mix start with "RDCM"
227+
* @return Whether given id belongs to a YouTube Channel Mix
228+
*/
229+
public static boolean isYoutubeChannelMixId(final String playlistId) {
230+
return playlistId.startsWith("RDCM");
231+
}
232+
233+
/**
234+
* Extracts the video id from the playlist id for Mixes.
235+
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
236+
*/
237+
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
238+
if (playlistId.startsWith("RDMM")) { //My Mix
239+
return playlistId.substring(4);
240+
241+
} else if (playlistId.startsWith("RDAMVM")) { //Music mix
242+
return playlistId.substring(6);
243+
244+
} else if (playlistId.startsWith("RMCM")) { //Channel mix
245+
//Channel mix are build with RMCM{channelId}, so videoId can't be determined
246+
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
247+
248+
} else if (playlistId.startsWith("RD")) { // Normal mix
249+
return playlistId.substring(2);
250+
251+
} else { //not a mix
252+
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
253+
}
254+
}
255+
195256
public static JsonObject getInitialData(String html) throws ParsingException {
196257
try {
197258
try {
@@ -416,10 +477,14 @@ public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint)
416477
} else if (navigationEndpoint.has("watchEndpoint")) {
417478
StringBuilder url = new StringBuilder();
418479
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
419-
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
420-
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
421-
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
422-
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
480+
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
481+
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint")
482+
.getString("playlistId"));
483+
}
484+
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) {
485+
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint")
486+
.getInt("startTimeSeconds"));
487+
}
423488
return url.toString();
424489
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
425490
return "https://www.youtube.com/playlist?list=" +
@@ -485,8 +550,8 @@ public static String fixThumbnailUrl(String thumbnailUrl) {
485550
public static String getValidJsonResponseBody(final Response response)
486551
throws ParsingException, MalformedURLException {
487552
if (response.responseCode() == 404) {
488-
throw new ContentNotAvailableException("Not found" +
489-
" (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
553+
throw new ContentNotAvailableException("Not found"
554+
+ " (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
490555
}
491556

492557
final String responseBody = response.responseBody();
@@ -506,22 +571,64 @@ public static String getValidJsonResponseBody(final Response response)
506571
final String responseContentType = response.getHeader("Content-Type");
507572
if (responseContentType != null
508573
&& responseContentType.toLowerCase().contains("text/html")) {
509-
throw new ParsingException("Got HTML document, expected JSON response" +
510-
" (latest url was: \"" + response.latestUrl() + "\")");
574+
throw new ParsingException("Got HTML document, expected JSON response"
575+
+ " (latest url was: \"" + response.latestUrl() + "\")");
511576
}
512577

513578
return responseBody;
514579
}
515580

581+
public static Response getResponse(final String url, final Localization localization)
582+
throws IOException, ExtractionException {
583+
final Map<String, List<String>> headers = new HashMap<>();
584+
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
585+
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
586+
587+
final Response response = getDownloader().get(url, headers, localization);
588+
getValidJsonResponseBody(response);
589+
590+
return response;
591+
}
592+
593+
public static String extractCookieValue(final String cookieName, final Response response) {
594+
final List<String> cookies = response.responseHeaders().get("Set-Cookie");
595+
int startIndex;
596+
String result = "";
597+
for (final String cookie : cookies) {
598+
startIndex = cookie.indexOf(cookieName);
599+
if (startIndex != -1) {
600+
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
601+
cookie.indexOf(";", startIndex));
602+
}
603+
}
604+
return result;
605+
}
606+
516607
public static JsonArray getJsonResponse(final String url, final Localization localization)
517608
throws IOException, ExtractionException {
518609
Map<String, List<String>> headers = new HashMap<>();
519610
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
520611
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
521612
final Response response = getDownloader().get(url, headers, localization);
522613

523-
final String responseBody = getValidJsonResponseBody(response);
614+
return toJsonArray(getValidJsonResponseBody(response));
615+
}
616+
617+
public static JsonArray getJsonResponse(final Page page, final Localization localization)
618+
throws IOException, ExtractionException {
619+
final Map<String, List<String>> headers = new HashMap<>();
620+
if (!isNullOrEmpty(page.getCookies())) {
621+
headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
622+
}
623+
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
624+
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
625+
626+
final Response response = getDownloader().get(page.getUrl(), headers, localization);
627+
628+
return toJsonArray(getValidJsonResponseBody(response));
629+
}
524630

631+
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
525632
try {
526633
return JsonParser.array().from(responseBody);
527634
} catch (JsonParserException e) {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
2121
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
2222
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
23+
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
2324
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor;
2425
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
2526
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
@@ -109,8 +110,12 @@ public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
109110
}
110111

111112
@Override
112-
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
113-
return new YoutubePlaylistExtractor(this, linkHandler);
113+
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
114+
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
115+
return new YoutubeMixPlaylistExtractor(this, linkHandler);
116+
} else {
117+
return new YoutubePlaylistExtractor(this, linkHandler);
118+
}
114119
}
115120

116121
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.grack.nanojson.JsonObject;
44

5+
import org.schabi.newpipe.extractor.ListExtractor;
56
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
67
import org.schabi.newpipe.extractor.exceptions.ParsingException;
78
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
@@ -86,7 +87,7 @@ public long getStreamCount() throws ParsingException {
8687
try {
8788
if (!channelInfoItem.has("videoCountText")) {
8889
// Video count is not available, channel probably has no public uploads.
89-
return -1;
90+
return ListExtractor.ITEM_COUNT_UNKNOWN;
9091
}
9192

9293
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(

0 commit comments

Comments
 (0)