Skip to content

Commit 907fc2a

Browse files
litetexlitetex
committed
Fix compile problems and optimize TokenStream
1 parent bcf24de commit 907fc2a

1 file changed

Lines changed: 18 additions & 82 deletions

File tree

  • extractor/src/main/java/org/schabi/newpipe/extractor/utils/jsextractor

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/jsextractor/TokenStream.java‎

Lines changed: 18 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import org.mozilla.javascript.Context;
44
import org.mozilla.javascript.Kit;
5-
import org.mozilla.javascript.ObjToIntMap;
65
import org.mozilla.javascript.ScriptRuntime;
76
import org.schabi.newpipe.extractor.exceptions.ParsingException;
87

@@ -38,12 +37,10 @@ class TokenStream {
3837
this.languageVersion = languageVersion;
3938
}
4039

41-
static boolean isKeyword(final String s, final int version, final boolean isStrict) {
42-
return Token.EOF != stringToKeyword(s, version, isStrict);
43-
}
44-
45-
private static Token stringToKeyword(final String name, final int version,
46-
final boolean isStrict) {
40+
private static Token stringToKeyword(
41+
final String name,
42+
final int version,
43+
final boolean isStrict) {
4744
if (version < Context.VERSION_ES6) {
4845
return stringToKeywordForJS(name);
4946
}
@@ -343,7 +340,7 @@ final Token getToken() throws ParsingException {
343340
}
344341
ungetChar(c);
345342

346-
String str = getStringFromBuffer();
343+
final String str = getStringFromBuffer();
347344
if (!containsEscape) {
348345
// OPT we shouldn't have to make a string (object!) to
349346
// check if it's a keyword.
@@ -353,30 +350,17 @@ final Token getToken() throws ParsingException {
353350
if (result != Token.EOF) {
354351
if ((result == Token.LET || result == Token.YIELD)
355352
&& languageVersion < Context.VERSION_1_7) {
356-
// LET and YIELD are tokens only in 1.7 and later
357-
string = result == Token.LET ? "let" : "yield";
358353
result = Token.NAME;
359354
}
360355
// Save the string in case we need to use in
361356
// object literal definitions.
362-
this.string = (String) allStrings.intern(str);
363-
if (result != Token.RESERVED) {
364-
return result;
365-
} else if (languageVersion >= Context.VERSION_ES6) {
366-
return result;
367-
} else if (!IS_RESERVED_KEYWORD_AS_IDENTIFIER) {
357+
if (result != Token.RESERVED
358+
|| languageVersion >= Context.VERSION_ES6
359+
|| !IS_RESERVED_KEYWORD_AS_IDENTIFIER) {
368360
return result;
369361
}
370362
}
371-
} else if (isKeyword(
372-
str,
373-
languageVersion,
374-
STRICT_MODE)) {
375-
// If a string contains unicodes, and converted to a keyword,
376-
// we convert the last character back to unicode
377-
str = convertLastCharToHex(str);
378363
}
379-
this.string = (String) allStrings.intern(str);
380364
return Token.NAME;
381365
}
382366

@@ -466,7 +450,7 @@ final Token getToken() throws ParsingException {
466450
}
467451
}
468452
ungetChar(c);
469-
this.string = getStringFromBuffer();
453+
tokenEnd = cursor;
470454
return Token.NUMBER;
471455
}
472456

@@ -562,15 +546,15 @@ final Token getToken() throws ParsingException {
562546
escapeVal = Kit.xDigitToInt(c, 0);
563547
if (escapeVal < 0) {
564548
addToString('x');
565-
continue strLoop;
549+
continue;
566550
}
567551
final int c1 = c;
568552
c = getChar();
569553
escapeVal = Kit.xDigitToInt(c, escapeVal);
570554
if (escapeVal < 0) {
571555
addToString('x');
572556
addToString(c1);
573-
continue strLoop;
557+
continue;
574558
}
575559
// got 2 hex digits
576560
c = escapeVal;
@@ -580,7 +564,7 @@ final Token getToken() throws ParsingException {
580564
// Remove line terminator after escape to follow
581565
// SpiderMonkey and C/C++
582566
c = getChar();
583-
continue strLoop;
567+
continue;
584568

585569
default:
586570
if ('0' <= c && c < '8') {
@@ -605,8 +589,7 @@ final Token getToken() throws ParsingException {
605589
c = getChar(false);
606590
}
607591

608-
final String str = getStringFromBuffer();
609-
this.string = (String) allStrings.intern(str);
592+
tokenEnd = cursor;
610593
return quoteChar == '`' ? Token.TEMPLATE_LITERAL : Token.STRING;
611594
}
612595

@@ -722,14 +705,13 @@ final Token getToken() throws ParsingException {
722705
return Token.GT;
723706

724707
case '*':
725-
if (languageVersion >= Context.VERSION_ES6) {
726-
if (matchChar('*')) {
727-
if (matchChar('=')) {
728-
return Token.ASSIGN_EXP;
729-
}
730-
return Token.EXP;
708+
if (languageVersion >= Context.VERSION_ES6 && matchChar('*')) {
709+
if (matchChar('=')) {
710+
return Token.ASSIGN_EXP;
731711
}
712+
return Token.EXP;
732713
}
714+
733715
if (matchChar('=')) {
734716
return Token.ASSIGN_MUL;
735717
}
@@ -920,7 +902,6 @@ void readRegExp(final Token startToken) throws ParsingException {
920902
}
921903
if (peekChar() == '*') {
922904
tokenEnd = cursor - 1;
923-
this.string = new String(stringBuffer, 0, stringBufferTop);
924905
throw new ParsingException("msg.unterminated.re.lit");
925906
}
926907
}
@@ -944,7 +925,6 @@ void readRegExp(final Token startToken) throws ParsingException {
944925
}
945926
addToString(c);
946927
}
947-
final int reEnd = stringBufferTop;
948928

949929
while (true) {
950930
c = getCharIgnoreLineEnd();
@@ -959,7 +939,6 @@ void readRegExp(final Token startToken) throws ParsingException {
959939
}
960940

961941
tokenEnd = start + stringBufferTop + 2; // include slashes
962-
this.string = new String(stringBuffer, 0, reEnd);
963942
}
964943

965944
private String getStringFromBuffer() {
@@ -1019,7 +998,6 @@ private int getChar(final boolean skipFormattingChars, final boolean ignoreLineE
1019998

1020999
for (;;) {
10211000
if (sourceCursor == sourceString.length()) {
1022-
hitEOF = true;
10231001
return EOF_CHAR;
10241002
}
10251003
cursor++;
@@ -1031,7 +1009,6 @@ private int getChar(final boolean skipFormattingChars, final boolean ignoreLineE
10311009
continue;
10321010
}
10331011
lineEndChar = -1;
1034-
lineStart = sourceCursor - 1;
10351012
lineno++;
10361013
}
10371014

@@ -1078,42 +1055,6 @@ private void skipLine() {
10781055
tokenEnd = cursor;
10791056
}
10801057

1081-
/** Return the current position of the scanner cursor. */
1082-
public int getCursor() {
1083-
return cursor;
1084-
}
1085-
1086-
/** Return the absolute source offset of the last scanned token. */
1087-
public int getTokenBeg() {
1088-
return tokenBeg;
1089-
}
1090-
1091-
/** Return the absolute source end-offset of the last scanned token. */
1092-
public int getTokenEnd() {
1093-
return tokenEnd;
1094-
}
1095-
1096-
/** Return tokenEnd - tokenBeg */
1097-
public int getTokenLength() {
1098-
return tokenEnd - tokenBeg;
1099-
}
1100-
1101-
public String getTokenRaw() {
1102-
return sourceString.substring(tokenBeg, tokenEnd);
1103-
}
1104-
1105-
private static String convertLastCharToHex(final String str) {
1106-
final int lastIndex = str.length() - 1;
1107-
final StringBuilder buf = new StringBuilder(str.substring(0, lastIndex));
1108-
buf.append("\\u");
1109-
final String hexCode = Integer.toHexString(str.charAt(lastIndex));
1110-
for (int i = 0; i < 4 - hexCode.length(); ++i) {
1111-
buf.append('0');
1112-
}
1113-
buf.append(hexCode);
1114-
return buf.toString();
1115-
}
1116-
11171058
public Token nextToken() throws ParsingException {
11181059
Token tt = getToken();
11191060
while (tt == Token.EOL || tt == Token.COMMENT) {
@@ -1124,19 +1065,14 @@ public Token nextToken() throws ParsingException {
11241065

11251066
// stuff other than whitespace since start of line
11261067
private boolean dirtyLine;
1127-
private String string = "";
11281068

11291069
private char[] stringBuffer = new char[128];
11301070
private int stringBufferTop;
1131-
private final ObjToIntMap allStrings = new ObjToIntMap(50);
11321071

11331072
// Room to backtrace from to < on failed match of the last - in <!--
11341073
private final int[] ungetBuffer = new int[3];
11351074
private int ungetCursor;
11361075

1137-
private boolean hitEOF = false;
1138-
1139-
private int lineStart = 0;
11401076
private int lineEndChar = -1;
11411077
int lineno;
11421078

0 commit comments

Comments
 (0)