Skip to content

Commit c97a19d

Browse files
StypoxStypox
authored
Merge pull request #696 from XiangRongLin/decrypt_pattern
Expand regex to match n param decrypt function
2 parents 06a5219 + 37df225 commit c97a19d

5 files changed

Lines changed: 157 additions & 10 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import org.schabi.newpipe.extractor.exceptions.ParsingException;
44
import org.schabi.newpipe.extractor.utils.JavaScript;
55
import org.schabi.newpipe.extractor.utils.Parser;
6+
import org.schabi.newpipe.extractor.utils.StringUtils;
67

78
import javax.annotation.Nonnull;
89
import java.util.HashMap;
@@ -33,7 +34,10 @@
3334
*/
3435
public class YoutubeThrottlingDecrypter {
3536

36-
private static final String N_PARAM_REGEX = "[&?]n=([^&]+)";
37+
private static final Pattern N_PARAM_PATTERN = Pattern.compile("[&?]n=([^&]+)");
38+
private static final Pattern FUNCTION_NAME_PATTERN = Pattern.compile(
39+
"b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)");
40+
3741
private static final Map<String, String> nParams = new HashMap<>();
3842

3943
private final String functionName;
@@ -62,15 +66,26 @@ public YoutubeThrottlingDecrypter() throws ParsingException {
6266

6367
private String parseDecodeFunctionName(final String playerJsCode)
6468
throws Parser.RegexException {
65-
Pattern pattern = Pattern.compile(
66-
"b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)");
67-
return Parser.matchGroup1(pattern, playerJsCode);
69+
return Parser.matchGroup1(FUNCTION_NAME_PATTERN, playerJsCode);
6870
}
6971

7072
@Nonnull
7173
private String parseDecodeFunction(final String playerJsCode, final String functionName)
7274
throws Parser.RegexException {
73-
Pattern functionPattern = Pattern.compile(functionName + "=function(.*?;)\n",
75+
try {
76+
return parseWithParenthesisMatching(playerJsCode, functionName);
77+
} catch (Exception e) {
78+
return parseWithRegex(playerJsCode, functionName);
79+
}
80+
}
81+
82+
private String parseWithParenthesisMatching(final String playerJsCode, final String functionName) {
83+
final String functionBase = functionName + "=function";
84+
return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase) + ";";
85+
}
86+
87+
private String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException {
88+
Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n",
7489
Pattern.DOTALL);
7590
return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode);
7691
}
@@ -86,12 +101,11 @@ public String apply(final String url) throws Parser.RegexException {
86101
}
87102

88103
private boolean containsNParam(final String url) {
89-
return Parser.isMatch(N_PARAM_REGEX, url);
104+
return Parser.isMatch(N_PARAM_PATTERN, url);
90105
}
91106

92107
private String parseNParam(final String url) throws Parser.RegexException {
93-
Pattern nValuePattern = Pattern.compile(N_PARAM_REGEX);
94-
return Parser.matchGroup1(nValuePattern, url);
108+
return Parser.matchGroup1(N_PARAM_PATTERN, url);
95109
}
96110

97111
private String decryptNParam(final String nParam) {

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,13 @@ public static String matchGroup(Pattern pat, String input, int group) throws Reg
7979
}
8080

8181
public static boolean isMatch(String pattern, String input) {
82-
Pattern pat = Pattern.compile(pattern);
83-
Matcher mat = pat.matcher(input);
82+
final Pattern pat = Pattern.compile(pattern);
83+
final Matcher mat = pat.matcher(input);
84+
return mat.find();
85+
}
86+
87+
public static boolean isMatch(Pattern pattern, String input) {
88+
final Matcher mat = pattern.matcher(input);
8489
return mat.find();
8590
}
8691

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/StringUtils.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package org.schabi.newpipe.extractor.utils;
2+
3+
4+
import javax.annotation.Nonnull;
5+
6+
public class StringUtils {
7+
8+
private StringUtils() {
9+
}
10+
11+
/**
12+
* @param string The string to search in.
13+
* @param start A string from which to start searching.
14+
* @return A substring where each '{' matches a '}'.
15+
* @throws IndexOutOfBoundsException If {@code string} does not contain {@code start}
16+
* or parenthesis could not be matched .
17+
*/
18+
@Nonnull
19+
public static String matchToClosingParenthesis(@Nonnull final String string, @Nonnull final String start) {
20+
int startIndex = string.indexOf(start);
21+
if (startIndex < 0) {
22+
throw new IndexOutOfBoundsException();
23+
}
24+
25+
startIndex += start.length();
26+
int endIndex = startIndex;
27+
while (string.charAt(endIndex) != '{') {
28+
++endIndex;
29+
}
30+
++endIndex;
31+
32+
int openParenthesis = 1;
33+
while (openParenthesis > 0) {
34+
switch (string.charAt(endIndex)) {
35+
case '{':
36+
++openParenthesis;
37+
break;
38+
case '}':
39+
--openParenthesis;
40+
break;
41+
default:
42+
break;
43+
}
44+
++endIndex;
45+
}
46+
47+
return string.substring(startIndex, endIndex);
48+
}
49+
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import org.junit.Before;
44
import org.junit.Test;
5+
import org.mozilla.javascript.EvaluatorException;
56
import org.schabi.newpipe.downloader.DownloaderTestImpl;
67
import org.schabi.newpipe.extractor.NewPipe;
78
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -11,6 +12,7 @@
1112
import static org.hamcrest.CoreMatchers.equalTo;
1213
import static org.hamcrest.MatcherAssert.assertThat;
1314
import static org.junit.Assert.assertNotEquals;
15+
import static org.junit.Assert.fail;
1416

1517
public class YoutubeThrottlingDecrypterTest {
1618

@@ -19,6 +21,22 @@ public void setup() throws IOException {
1921
NewPipe.init(DownloaderTestImpl.getInstance());
2022
}
2123

24+
@Test
25+
public void testExtractFunction__success() throws ParsingException {
26+
final String[] videoIds = {"jE1USQrs1rw", "CqxjzfudGAc", "goH-9MfQI7w", "KYIdr_7H5Yw", "J1WeqmGbYeI"};
27+
28+
final String encryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190";
29+
30+
for (final String videoId : videoIds) {
31+
try {
32+
final String decryptedUrl = new YoutubeThrottlingDecrypter(videoId).apply(encryptedUrl);
33+
assertNotEquals(encryptedUrl, decryptedUrl);
34+
} catch (EvaluatorException e) {
35+
fail("Failed to extract n param decrypt function for video " + videoId + "\n" + e);
36+
}
37+
}
38+
}
39+
2240
@Test
2341
public void testDecode__success() throws ParsingException {
2442
// URL extracted from browser with the dev tools

‎extractor/src/test/java/org/schabi/newpipe/extractor/utils/StringUtilsTest.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package org.schabi.newpipe.extractor.utils;
2+
3+
import org.junit.Ignore;
4+
import org.junit.Test;
5+
6+
import static org.junit.Assert.assertEquals;
7+
import static org.schabi.newpipe.extractor.utils.StringUtils.matchToClosingParenthesis;
8+
9+
public class StringUtilsTest {
10+
11+
@Test
12+
public void actualDecodeFunction__success() {
13+
String preNoise = "if(\"function\"===typeof b&&\"function\"===typeof c||\"function\"===typeof c&&\"function\"===typeof d)throw Error(\"It looks like you are passing several store enhancers to createStore(). This is not supported. Instead, compose them together to a single function.\");\"function\"===typeof b&&\"undefined\"===typeof c&&(c=b,b=void 0);if(\"undefined\"!==typeof c){if(\"function\"!==typeof c)throw Error(\"Expected the enhancer to be a function.\");return c(Dr)(a,b)}if(\"function\"!==typeof a)throw Error(\"Expected the reducer to be a function.\");\n" +
14+
"var l=a,m=b,n=[],p=n,q=!1;h({type:Cr});a={};var t=(a.dispatch=h,a.subscribe=f,a.getState=e,a.replaceReducer=function(u){if(\"function\"!==typeof u)throw Error(\"Expected the nextReducer to be a function.\");l=u;h({type:hha});return t},a[Er]=function(){var u={};\n" +
15+
"return u.subscribe=function(x){function y(){x.next&&x.next(e())}\n" +
16+
"if(\"object\"!==typeof x||null===x)throw new TypeError(\"Expected the observer to be an object.\");y();return{unsubscribe:f(y)}},u[Er]=function(){return this},u},a);\n" +
17+
"return t};\n" +
18+
"Fr=function(a){De.call(this,a,-1,iha)};\n" +
19+
"Gr=function(a){De.call(this,a)};\n" +
20+
"jha=function(a,b){for(;Jd(b);)switch(b.C){case 10:var c=Od(b);Ge(a,1,c);break;case 18:c=Od(b);Ge(a,2,c);break;case 26:c=Od(b);Ge(a,3,c);break;case 34:c=Od(b);Ge(a,4,c);break;case 40:c=Hd(b.i);Ge(a,5,c);break;default:if(!we(b))return a}return a};";
21+
String signature = "kha=function(a)";
22+
String body = "{var b=a.split(\"\"),c=[-1186681497,-1653318181,372630254,function(d,e){for(var f=64,h=[];++f-h.length-32;){switch(f){case 58:f-=14;case 91:case 92:case 93:continue;case 123:f=47;case 94:case 95:case 96:continue;case 46:f=95}h.push(String.fromCharCode(f))}d.forEach(function(l,m,n){this.push(n[m]=h[(h.indexOf(l)-h.indexOf(this[m])+m-32+f--)%h.length])},e.split(\"\"))},\n" +
23+
"-467738125,1158037010,function(d,e){e=(e%d.length+d.length)%d.length;var f=d[0];d[0]=d[e];d[e]=f},\n" +
24+
"\"continue\",158531598,-172776392,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})},\n" +
25+
"-1753359936,function(d){for(var e=d.length;e;)d.push(d.splice(--e,1)[0])},\n" +
26+
"1533713399,-1736576025,-1274201783,function(d){d.reverse()},\n" +
27+
"169126570,1077517431,function(d,e){d.push(e)},\n" +
28+
"-1807932259,-150219E3,480561184,-3495188,-1856307605,1416497372,b,-1034568435,-501230371,1979778585,null,b,-1049521459,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])},\n" +
29+
"1119056651,function(d,e){for(e=(e%d.length+d.length)%d.length;e--;)d.unshift(d.pop())},\n" +
30+
"b,1460920438,135616752,-1807932259,-815823682,-387465417,1979778585,113585E4,function(d,e){d.push(e)},\n" +
31+
"-1753359936,-241651400,-386043301,-144139513,null,null,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(e,1)}];\n" +
32+
"c[30]=c;c[49]=c;c[50]=c;try{c[51](c[26],c[25]),c[10](c[30],c[17]),c[5](c[28],c[9]),c[18](c[51]),c[14](c[19],c[21]),c[8](c[40],c[22]),c[50](c[35],c[28]),c[24](c[29],c[3]),c[0](c[31],c[19]),c[27](c[26],c[33]),c[29](c[36],c[40]),c[50](c[26]),c[27](c[32],c[9]),c[8](c[10],c[14]),c[35](c[44],c[28]),c[22](c[44],c[1]),c[8](c[11],c[3]),c[29](c[44]),c[21](c[41],c[45]),c[16](c[32],c[4]),c[17](c[14],c[26]),c[36](c[20],c[45]),c[43](c[35],c[39]),c[43](c[20],c[23]),c[43](c[10],c[51]),c[43](c[34],c[32]),c[29](c[34],\n" +
33+
"c[49]),c[43](c[20],c[44]),c[49](c[20]),c[19](c[15],c[8]),c[36](c[15],c[46]),c[17](c[20],c[37]),c[18](c[10]),c[17](c[34],c[31]),c[19](c[10],c[30]),c[19](c[20],c[2]),c[36](c[20],c[21]),c[43](c[35],c[16]),c[19](c[35],c[5]),c[18](c[46],c[34])}catch(d){return\"enhanced_except_lJMB6-z-_w8_\"+a}return b.join(\"\")}";
34+
String postNoise = "Hr=function(a){this.i=a}";
35+
36+
String substring = matchToClosingParenthesis(preNoise + '\n' + signature + body + ";" + postNoise, signature);
37+
38+
assertEquals(body, substring);
39+
}
40+
41+
@Test
42+
public void moreClosing__success() {
43+
String expected = "{{{}}}";
44+
String string = "a" + expected + "}}";
45+
46+
String substring = matchToClosingParenthesis(string, "a");
47+
48+
assertEquals(expected, substring);
49+
}
50+
51+
@Ignore("Functionality currently not needed")
52+
@Test
53+
public void lessClosing__success() {
54+
String expected = "{{{}}}";
55+
String string = "a{{" + expected;
56+
57+
String substring = matchToClosingParenthesis(string, "a");
58+
59+
assertEquals(expected, substring);
60+
}
61+
}

0 commit comments

Comments
 (0)