Skip to content

Commit d8279f9

Browse files
KoitharuKoitharu
committed
Merge branch 'dev' of https://github.com/TeamNewPipe/NewPipeExtractor into feature/frames
2 parents ecb8ad8 + 8ab48c6 commit d8279f9

14 files changed

Lines changed: 407 additions & 107 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
public class YoutubeChannelExtractor extends ChannelExtractor {
5050
/*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
5151
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
52-
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
52+
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000&gl=US&hl=en";
5353

5454
private Document doc;
5555

@@ -82,6 +82,11 @@ public String getUrl() throws ParsingException {
8282
@Nonnull
8383
@Override
8484
public String getId() throws ParsingException {
85+
try {
86+
return doc.select("meta[itemprop=\"channelId\"]").first().attr("content");
87+
} catch (Exception ignored) {}
88+
89+
// fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO)
8590
try {
8691
Element element = doc.getElementsByClass("yt-uix-subscription-button").first();
8792
if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first();
@@ -135,10 +140,12 @@ public String getFeedUrl() throws ParsingException {
135140

136141
@Override
137142
public long getSubscriberCount() throws ParsingException {
143+
138144
final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
139145
if (el != null) {
146+
String elTitle = el.attr("title");
140147
try {
141-
return Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
148+
return Utils.mixedNumberWordToLong(elTitle);
142149
} catch (NumberFormatException e) {
143150
throw new ParsingException("Could not get subscriber count", e);
144151
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -56,19 +56,25 @@ public String getName() throws ParsingException {
5656

5757
@Override
5858
public String getUrl() throws ParsingException {
59-
String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
60-
.attr("abs:data-href");
59+
try {
60+
String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
61+
.attr("abs:data-href");
6162

62-
Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
63-
Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
63+
Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
64+
Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
6465

65-
if (match.matches()) {
66-
return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
67-
} else {
68-
// fallback method just in case youtube changes things; it should never run and tests will fail
69-
// provides an url with "/user/NAME", that is inconsistent with stream and channel extractor
66+
if (match.matches()) {
67+
return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
68+
}
69+
} catch(Exception ignored) {}
70+
71+
// fallback method for channels without "Subscribe" button (or just in case yt changes things)
72+
// provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail
73+
try {
7074
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
7175
.attr("abs:href");
76+
} catch (Exception e) {
77+
throw new ParsingException("Could not get channel url", e);
7278
}
7379
}
7480

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 71 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ public class SubtitlesException extends ContentNotAvailableException {
8585
private JsonObject playerArgs;
8686
@Nonnull
8787
private final Map<String, String> videoInfoPage = new HashMap<>();
88+
private JsonObject playerResponse;
8889

8990
@Nonnull
9091
private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
@@ -253,20 +254,6 @@ public int getAgeLimit() throws ParsingException {
253254
public long getLength() throws ParsingException {
254255
assertPageFetched();
255256

256-
final JsonObject playerResponse;
257-
try {
258-
final String pr;
259-
if(playerArgs != null) {
260-
pr = playerArgs.getString("player_response");
261-
} else {
262-
pr = videoInfoPage.get("player_response");
263-
}
264-
playerResponse = JsonParser.object()
265-
.from(pr);
266-
} catch (Exception e) {
267-
throw new ParsingException("Could not get playerResponse", e);
268-
}
269-
270257
// try getting duration from playerargs
271258
try {
272259
String durationMs = playerResponse
@@ -442,31 +429,24 @@ public String getDashMpdUrl() throws ParsingException {
442429
@Override
443430
public String getHlsUrl() throws ParsingException {
444431
assertPageFetched();
445-
try {
446-
String hlsvp = "";
447-
if (playerArgs != null) {
448-
if( playerArgs.isString("hlsvp") ) {
449-
hlsvp = playerArgs.getString("hlsvp", "");
450-
}else {
451-
hlsvp = JsonParser.object()
452-
.from(playerArgs.getString("player_response", "{}"))
453-
.getObject("streamingData", new JsonObject())
454-
.getString("hlsManifestUrl", "");
455-
}
456-
}
457432

458-
return hlsvp;
433+
try {
434+
return playerResponse.getObject("streamingData").getString("hlsManifestUrl");
459435
} catch (Exception e) {
460-
throw new ParsingException("Could not get hls manifest url", e);
436+
if (playerArgs != null && playerArgs.isString("hlsvp")) {
437+
return playerArgs.getString("hlsvp");
438+
} else {
439+
throw new ParsingException("Could not get hls manifest url", e);
440+
}
461441
}
462442
}
463443

464444
@Override
465-
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
445+
public List<AudioStream> getAudioStreams() throws ExtractionException {
466446
assertPageFetched();
467447
List<AudioStream> audioStreams = new ArrayList<>();
468448
try {
469-
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.AUDIO).entrySet()) {
449+
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) {
470450
ItagItem itag = entry.getValue();
471451

472452
AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate);
@@ -482,11 +462,11 @@ public List<AudioStream> getAudioStreams() throws IOException, ExtractionExcepti
482462
}
483463

484464
@Override
485-
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
465+
public List<VideoStream> getVideoStreams() throws ExtractionException {
486466
assertPageFetched();
487467
List<VideoStream> videoStreams = new ArrayList<>();
488468
try {
489-
for (Map.Entry<String, ItagItem> entry : getItags(URL_ENCODED_FMT_STREAM_MAP, ItagItem.ItagType.VIDEO).entrySet()) {
469+
for (Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
490470
ItagItem itag = entry.getValue();
491471

492472
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString);
@@ -506,7 +486,7 @@ public List<VideoStream> getVideoOnlyStreams() throws ExtractionException {
506486
assertPageFetched();
507487
List<VideoStream> videoOnlyStreams = new ArrayList<>();
508488
try {
509-
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
489+
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
510490
ItagItem itag = entry.getValue();
511491

512492
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true);
@@ -543,7 +523,7 @@ public StreamType getStreamType() throws ParsingException {
543523
assertPageFetched();
544524
try {
545525
if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
546-
playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
526+
(!playerResponse.getObject("streamingData").has(FORMATS)))) {
547527
return StreamType.LIVE_STREAM;
548528
}
549529
} catch (Exception e) {
@@ -595,21 +575,26 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti
595575
*/
596576
@Override
597577
public String getErrorMessage() {
598-
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
599578
StringBuilder errorReason;
579+
Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
600580

601-
if (errorMessage == null || errorMessage.isEmpty()) {
581+
if (errorElement == null) {
602582
errorReason = null;
603-
} else if (errorMessage.contains("GEMA")) {
604-
// Gema sometimes blocks youtube music content in germany:
605-
// https://www.gema.de/en/
606-
// Detailed description:
607-
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
608-
errorReason = new StringBuilder("GEMA");
609583
} else {
610-
errorReason = new StringBuilder(errorMessage);
611-
errorReason.append(" ");
612-
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
584+
String errorMessage = errorElement.text();
585+
if (errorMessage == null || errorMessage.isEmpty()) {
586+
errorReason = null;
587+
} else if (errorMessage.contains("GEMA")) {
588+
// Gema sometimes blocks youtube music content in germany:
589+
// https://www.gema.de/en/
590+
// Detailed description:
591+
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
592+
errorReason = new StringBuilder("GEMA");
593+
} else {
594+
errorReason = new StringBuilder(errorMessage);
595+
errorReason.append(" ");
596+
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
597+
}
613598
}
614599

615600
return errorReason != null ? errorReason.toString() : null;
@@ -619,8 +604,8 @@ public String getErrorMessage() {
619604
// Fetch page
620605
//////////////////////////////////////////////////////////////////////////*/
621606

622-
private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
623-
private static final String ADAPTIVE_FMTS = "adaptive_fmts";
607+
private static final String FORMATS = "formats";
608+
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
624609
private static final String HTTPS = "https:";
625610
private static final String CONTENT = "content";
626611
private static final String DECRYPTION_FUNC_NAME = "decrypt";
@@ -667,6 +652,7 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
667652
playerUrl = getPlayerUrl(ytPlayerConfig);
668653
isAgeRestricted = false;
669654
}
655+
playerResponse = getPlayerResponse();
670656

671657
if (decryptionCode.isEmpty()) {
672658
decryptionCode = loadDecryptionCode(playerUrl);
@@ -728,6 +714,20 @@ private String getPlayerUrl(JsonObject playerConfig) throws ParsingException {
728714
}
729715
}
730716

717+
private JsonObject getPlayerResponse() throws ParsingException {
718+
try {
719+
String playerResponseStr;
720+
if(playerArgs != null) {
721+
playerResponseStr = playerArgs.getString("player_response");
722+
} else {
723+
playerResponseStr = videoInfoPage.get("player_response");
724+
}
725+
return JsonParser.object().from(playerResponseStr);
726+
} catch (Exception e) {
727+
throw new ParsingException("Could not parse yt player response", e);
728+
}
729+
}
730+
731731
@Nonnull
732732
private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException {
733733
try {
@@ -843,19 +843,13 @@ private List<SubtitlesInfo> getAvailableSubtitlesInfo() throws SubtitlesExceptio
843843
} catch (IOException | ExtractionException e) {
844844
throw new SubtitlesException("Unable to download player configs", e);
845845
}
846-
final String playerResponse = playerConfig.getObject("args", new JsonObject())
847-
.getString("player_response");
848846

849847
final JsonObject captions;
850-
try {
851-
if (playerResponse == null || !JsonParser.object().from(playerResponse).has("captions")) {
852-
// Captions does not exist
853-
return Collections.emptyList();
854-
}
855-
captions = JsonParser.object().from(playerResponse).getObject("captions");
856-
} catch (JsonParserException e) {
857-
throw new SubtitlesException("Unable to parse subtitles listing", e);
848+
if (!playerResponse.has("captions")) {
849+
// Captions does not exist
850+
return Collections.emptyList();
858851
}
852+
captions = playerResponse.getObject("captions");
859853

860854
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject());
861855
final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray());
@@ -924,45 +918,36 @@ private static String getVideoInfoUrl(final String id, final String sts) {
924918
"&sts=" + sts + "&ps=default&gl=US&hl=en";
925919
}
926920

927-
private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
921+
private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
928922
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
929-
930-
String encodedUrlMap = "";
931-
if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) {
932-
encodedUrlMap = playerArgs.getString(encodedUrlMapKey, "");
933-
} else if (videoInfoPage.containsKey(encodedUrlMapKey)) {
934-
encodedUrlMap = videoInfoPage.get(encodedUrlMapKey);
923+
JsonObject streamingData = playerResponse.getObject("streamingData");
924+
if (!streamingData.has(streamingDataKey)) {
925+
return urlAndItags;
935926
}
936927

937-
for (String url_data_str : encodedUrlMap.split(",")) {
938-
try {
939-
// This loop iterates through multiple streams, therefore tags
940-
// is related to one and the same stream at a time.
941-
Map<String, String> tags = Parser.compatParseMap(
942-
org.jsoup.parser.Parser.unescapeEntities(url_data_str, true));
943-
944-
int itag = Integer.parseInt(tags.get("itag"));
928+
JsonArray formats = streamingData.getArray(streamingDataKey);
929+
for (int i = 0; i != formats.size(); ++i) {
930+
JsonObject formatData = formats.getObject(i);
931+
int itag = formatData.getInt("itag");
945932

946-
if (ItagItem.isSupported(itag)) {
933+
if (ItagItem.isSupported(itag)) {
934+
try {
947935
ItagItem itagItem = ItagItem.getItag(itag);
948936
if (itagItem.itagType == itagTypeWanted) {
949-
String streamUrl = tags.get("url");
950-
// if video has a signature: decrypt it and add it to the url
951-
if (tags.get("s") != null) {
952-
if (tags.get("sp") == null) {
953-
// fallback for urls not conaining the "sp" tag
954-
streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode);
955-
}
956-
else {
957-
streamUrl = streamUrl + "&" + tags.get("sp") + "=" + decryptSignature(tags.get("s"), decryptionCode);
958-
}
937+
String streamUrl;
938+
if (formatData.has("url")) {
939+
streamUrl = formatData.getString("url");
940+
} else {
941+
// this url has an encrypted signature
942+
Map<String, String> cipher = Parser.compatParseMap(formatData.getString("cipher"));
943+
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + decryptSignature(cipher.get("s"), decryptionCode);
959944
}
945+
960946
urlAndItags.put(streamUrl, itagItem);
961947
}
948+
} catch (UnsupportedEncodingException ignored) {
949+
962950
}
963-
} catch (DecryptException e) {
964-
throw e;
965-
} catch (Exception ignored) {
966951
}
967952
}
968953

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ private YoutubeParsingHelper() {
3333
public static boolean isYoutubeURL(URL url) {
3434
String host = url.getHost();
3535
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
36-
|| host.equalsIgnoreCase("m.youtube.com");
36+
|| host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com");
3737
}
3838

3939
public static boolean isYoutubeServiceURL(URL url) {
@@ -48,7 +48,7 @@ public static boolean isHooktubeURL(URL url) {
4848

4949
public static boolean isInvidioURL(URL url) {
5050
String host = url.getHost();
51-
return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("www.invidio.us");
51+
return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("dev.invidio.us") || host.equalsIgnoreCase("www.invidio.us") || host.equalsIgnoreCase("invidious.snopyta.org") || host.equalsIgnoreCase("de.invidious.snopyta.org") || host.equalsIgnoreCase("fi.invidious.snopyta.org") || host.equalsIgnoreCase("vid.wxzm.sx") || host.equalsIgnoreCase("invidious.kabi.tk") || host.equalsIgnoreCase("invidiou.sh") || host.equalsIgnoreCase("www.invidiou.sh") || host.equalsIgnoreCase("no.invidiou.sh") || host.equalsIgnoreCase("invidious.enkirton.net") || host.equalsIgnoreCase("tube.poal.co") || host.equalsIgnoreCase("invidious.13ad.de") || host.equalsIgnoreCase("yt.elukerio.org");
5252
}
5353

5454
public static long parseDurationString(String input)

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,8 @@ public String getId(String urlString) throws ParsingException, IllegalArgumentEx
114114

115115
case "YOUTUBE.COM":
116116
case "WWW.YOUTUBE.COM":
117-
case "M.YOUTUBE.COM": {
117+
case "M.YOUTUBE.COM":
118+
case "MUSIC.YOUTUBE.COM": {
118119
if (path.equals("attribution_link")) {
119120
String uQueryValue = Utils.getQueryValue(url, "u");
120121

@@ -163,7 +164,20 @@ public String getId(String urlString) throws ParsingException, IllegalArgumentEx
163164
}
164165

165166
case "WWW.INVIDIO.US":
166-
case "INVIDIO.US": { // code-block for hooktube.com and invidio.us
167+
case "DEV.INVIDIO.US":
168+
case "INVIDIO.US":
169+
case "INVIDIOUS.SNOPYTA.ORG":
170+
case "DE.INVIDIOUS.SNOPYTA.ORG":
171+
case "FI.INVIDIOUS.SNOPYTA.ORG":
172+
case "VID.WXZM.SX":
173+
case "INVIDIOUS.KABI.TK":
174+
case "INVIDIOU.SH":
175+
case "WWW.INVIDIOU.SH":
176+
case "NO.INVIDIOU.SH":
177+
case "INVIDIOUS.ENKIRTON.NET":
178+
case "TUBE.POAL.CO":
179+
case "INVIDIOUS.13AD.DE":
180+
case "YT.ELUKERIO.ORG": { // code-block for hooktube.com and Invidious instances
167181
if (path.equals("watch")) {
168182
String viewQueryValue = Utils.getQueryValue(url, "v");
169183
if (viewQueryValue != null) {

0 commit comments

Comments
 (0)