mozilla
/
pjs
зеркало из https://github.com/mozilla/pjs.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Fixing http://bugzilla.mozilla.org/show_bug.cgi?id=191668 : 

Integration of LineBuffer into TokenStream code which now uses a special buffer for unreading of several chars to follow SM more closely. In this way there is no problem with a possible backtracking of 3 chars on failed attempt to match <!-- at the last minus.

TokenStream is also modified to accept a string with a source directly which avoids the need to construct intermediate StringReader in Context and allows to remove DebugReader class which is replaced by a simple function to read all Reader data into string.
This commit is contained in:
igor%mir2.org 2003-02-14 17:09:19 +00:00
Родитель cd30738141
Коммит edac1d4aee
5 изменённых файлов: 422 добавлений и 194 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

106
js/rhino/src/org/mozilla/javascript/Context.java
Просмотреть файл

@ -779,14 +779,12 @@ public class Context {
Object securityDomain) Object securityDomain)
throws JavaScriptException throws JavaScriptException
{ {
try { Script script = compileString(scope, source, sourceName, lineno,
Reader in = new StringReader(source);
return evaluateReader(scope, in, sourceName, lineno,
securityDomain); securityDomain);
} if (script != null) {
catch (IOException ioe) { return script.exec(this, scope);
// Should never occur because we just made the reader from a String } else {
throw new RuntimeException(); return null;
} }
} }
@ -816,11 +814,12 @@ public class Context {
{ {
Script script = compileReader(scope, in, sourceName, lineno, Script script = compileReader(scope, in, sourceName, lineno,
securityDomain); securityDomain);
if (script != null) if (script != null) {
return script.exec(this, scope); return script.exec(this, scope);
else } else {
return null; return null;
} }
}
/** /**
* Check whether a string is ready to be compiled. * Check whether a string is ready to be compiled.
@ -840,10 +839,9 @@ public class Context {
*/ */
synchronized public boolean stringIsCompilableUnit(String source) synchronized public boolean stringIsCompilableUnit(String source)
{ {
Reader in = new StringReader(source);
// no source name or source text manager, because we're just // no source name or source text manager, because we're just
// going to throw away the result. // going to throw away the result.
TokenStream ts = new TokenStream(in, null, null, 1); TokenStream ts = new TokenStream(null, source, null, null, 1);
// Temporarily set error reporter to always be the exception-throwing // Temporarily set error reporter to always be the exception-throwing
// DefaultErrorReporter. (This is why the method is synchronized...) // DefaultErrorReporter. (This is why the method is synchronized...)
@ -896,10 +894,41 @@ public class Context {
int lineno, Object securityDomain) int lineno, Object securityDomain)
throws IOException throws IOException
{ {
return (Script) compile(scope, in, sourceName, lineno, securityDomain, return (Script) compile(scope, in, null, sourceName, lineno,
false); securityDomain, false);
} }
/**
* Compiles the source in the given string.
* <p>
* Returns a script that may later be executed.
*
* @param scope if nonnull, will be the scope in which the script object
* is created. The script object will be a valid JavaScript object
* as if it were created using the JavaScript1.3 Script constructor
* @param source the source string
* @param sourceName a string describing the source, such as a filename
* @param lineno the starting line number for reporting errors
* @param securityDomain an arbitrary object that specifies security
* information about the origin or owner of the script. For
* implementations that don't care about security, this value
* may be null.
* @return a script that may later be executed
* @see org.mozilla.javascript.Script#exec
* @exception IOException if an IOException was generated by the Reader
*/
public Script compileString(Scriptable scope, String source,
String sourceName, int lineno,
Object securityDomain)
{
try {
return (Script) compile(scope, null, source, sourceName, lineno,
securityDomain, false);
} catch (IOException ex) {
// Should not happen when dealing with source as string
throw new RuntimeException();
}
}
/** /**
* Compile a JavaScript function. * Compile a JavaScript function.
@ -922,9 +951,8 @@ public class Context {
String sourceName, int lineno, String sourceName, int lineno,
Object securityDomain) Object securityDomain)
{ {
Reader in = new StringReader(source);
try { try {
return (Function) compile(scope, in, sourceName, lineno, return (Function) compile(scope, null, source, sourceName, lineno,
securityDomain, true); securityDomain, true);
} }
catch (IOException ioe) { catch (IOException ioe) {
@ -1904,6 +1932,24 @@ public class Context {
return formatter.format(arguments); return formatter.format(arguments);
} }
private static String readReader(Reader r)
throws IOException
{
char[] buffer = new char[512];
int cursor = 0;
for (;;) {
int n = r.read(buffer, cursor, buffer.length - cursor);
if (n < 0) { break; }
cursor += n;
if (cursor == buffer.length) {
char[] tmp = new char[buffer.length];
System.arraycopy(buffer, 0, tmp, 0, cursor);
buffer = tmp;
}
}
return new String(buffer, 0, cursor);
}
// debug flags // debug flags
static final boolean printTrees = false; static final boolean printTrees = false;
static final boolean printICode = false; static final boolean printICode = false;
@ -1930,22 +1976,30 @@ public class Context {
* @return a class for the script or function * @return a class for the script or function
* @see org.mozilla.javascript.Context#compileReader * @see org.mozilla.javascript.Context#compileReader
*/ */
private Object compile(Scriptable scope, Reader in, String sourceName, private Object compile(Scriptable scope,
int lineno, Object securityDomain, Reader sourceReader, String sourceString,
boolean returnFunction) String sourceName, int lineno,
Object securityDomain, boolean returnFunction)
throws IOException throws IOException
{ {
// One of sourceReader or sourceString has to be null
if (!(sourceReader == null ^ sourceString == null)) Context.codeBug();
Object dynamicDoamin = null; Object dynamicDoamin = null;
if (securityController != null) { if (securityController != null) {
dynamicDoamin = securityController. dynamicDoamin = securityController.
getDynamicSecurityDomain(securityDomain); getDynamicSecurityDomain(securityDomain);
} }
if (debugger != null && in != null) { if (debugger != null) {
in = new DebugReader(in); if (sourceReader != null) {
sourceString = readReader(sourceReader);
sourceReader = null;
} }
TokenStream ts = new TokenStream(in, scope, sourceName, lineno); }
return compile(scope, ts, dynamicDoamin, in, returnFunction); TokenStream ts = new TokenStream(sourceReader, sourceString,
scope, sourceName, lineno);
return compile(scope, ts, dynamicDoamin, sourceString, returnFunction);
} }
private static Class codegenClass; private static Class codegenClass;
@ -1978,7 +2032,7 @@ public class Context {
} }
private Object compile(Scriptable scope, TokenStream ts, private Object compile(Scriptable scope, TokenStream ts,
Object dynamicSecurityDomain, Reader in, Object dynamicSecurityDomain, String sourceString,
boolean returnFunction) boolean returnFunction)
throws IOException throws IOException
{ {
@ -2010,9 +2064,9 @@ public class Context {
return null; return null;
} }
if (in instanceof DebugReader) { if (debugger != null) {
DebugReader dr = (DebugReader) in; if (sourceString == null) Context.codeBug();
tree.putProp(Node.DEBUGSOURCE_PROP, dr.getSaved()); tree.putProp(Node.DEBUGSOURCE_PROP, sourceString);
} }
Object result = compiler.compile(this, scope, tree, Object result = compiler.compile(this, scope, tree,

0
js/rhino/src/org/mozilla/javascript/LineBuffer.java
Просмотреть файл

501
js/rhino/src/org/mozilla/javascript/TokenStream.java
Просмотреть файл

@ -21,6 +21,7 @@
* Contributor(s): * Contributor(s):
* Roger Lawrence * Roger Lawrence
* Mike McCabe * Mike McCabe
* Igor Bukanov
* *
* Alternatively, the contents of this file may be used under the * Alternatively, the contents of this file may be used under the
* terms of the GNU Public License (the "GPL"), in which case the * terms of the GNU Public License (the "GPL"), in which case the
@ -613,20 +614,45 @@ public class TokenStream {
return id & 0xff; return id & 0xff;
} }
public TokenStream(Reader in, Scriptable scope, public TokenStream(Reader sourceReader, String sourceString,
String sourceName, int lineno) Scriptable scope, String sourceName, int lineno)
{ {
this.in = new LineBuffer(in, lineno);
this.scope = scope; this.scope = scope;
this.pushbackToken = EOF; this.pushbackToken = EOF;
this.sourceName = sourceName; this.sourceName = sourceName;
flags = 0; this.lineno = lineno;
this.flags = 0;
if (sourceReader != null) {
if (sourceString != null) Context.codeBug();
this.sourceReader = sourceReader;
this.sourceBuffer = new char[512];
this.sourceEnd = 0;
} else {
if (sourceString == null) Context.codeBug();
this.sourceString = sourceString;
this.sourceEnd = sourceString.length();
}
this.sourceCursor = 0;
} }
public Scriptable getScope() { public Scriptable getScope() {
return scope; return scope;
} }
public String getSourceName() { return sourceName; }
public int getLineno() { return lineno; }
public int getOp() { return op; }
public String getString() { return string; }
public double getNumber() { return number; }
public int getTokenno() { return tokenno; }
public boolean eof() { return hitEOF; }
/* return and pop the token from the stream if it matches... /* return and pop the token from the stream if it matches...
* otherwise return null * otherwise return null
*/ */
@ -646,11 +672,8 @@ public class TokenStream {
} }
public void ungetToken(int tt) { public void ungetToken(int tt) {
if (this.pushbackToken != EOF && tt != ERROR) { // Can not unread more then one token
String message = Context.getMessage2("msg.token.replaces.pushback", if (this.pushbackToken != EOF && tt != ERROR) Context.codeBug();
tokenToString(tt), tokenToString(this.pushbackToken));
throw new RuntimeException(message);
}
this.pushbackToken = tt; this.pushbackToken = tt;
tokenno--; tokenno--;
} }
@ -674,68 +697,6 @@ public class TokenStream {
return result; return result;
} }
protected static boolean isJSIdentifier(String s) {
int length = s.length();
if (length == 0 || !Character.isJavaIdentifierStart(s.charAt(0)))
return false;
for (int i=1; i<length; i++) {
char c = s.charAt(i);
if (!Character.isJavaIdentifierPart(c))
if (c == '\\')
if (! ((i + 5) < length)
&& (s.charAt(i + 1) == 'u')
&& 0 <= xDigitToInt(s.charAt(i + 2))
&& 0 <= xDigitToInt(s.charAt(i + 3))
&& 0 <= xDigitToInt(s.charAt(i + 4))
&& 0 <= xDigitToInt(s.charAt(i + 5)))
return false;
}
return true;
}
private static boolean isAlpha(int c) {
return ((c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z'));
}
static boolean isDigit(int c) {
return (c >= '0' && c <= '9');
}
static int xDigitToInt(int c) {
if ('0' <= c && c <= '9') { return c - '0'; }
if ('a' <= c && c <= 'f') { return c - ('a' - 10); }
if ('A' <= c && c <= 'F') { return c - ('A' - 10); }
return -1;
}
/* As defined in ECMA. jsscan.c uses C isspace() (which allows
* \v, I think.) note that code in in.read() implicitly accepts
* '\r' == \u000D as well.
*/
public static boolean isJSSpace(int c) {
return (c == '\u0020' || c == '\u0009'
|| c == '\u000C' || c == '\u000B'
|| c == '\u00A0'
|| Character.getType((char)c) == Character.SPACE_SEPARATOR);
}
public static boolean isJSLineTerminator(int c) {
return (c == '\n' || c == '\r'
|| c == 0x2028 || c == 0x2029);
}
private void skipLine() throws IOException {
// skip to end of line
int c;
while ((c = in.read()) != EOF_CHAR && c != '\n') { }
in.unread();
}
public int getToken() throws IOException { public int getToken() throws IOException {
int c; int c;
tokenno++; tokenno++;
@ -748,34 +709,37 @@ public class TokenStream {
} }
// Eat whitespace, possibly sensitive to newlines. // Eat whitespace, possibly sensitive to newlines.
do { for (;;) {
c = in.read(); c = getChar();
if (c == '\n') { if (c == EOF_CHAR) {
return EOF;
} else if (c == '\n') {
flags &= ~TSF_DIRTYLINE; flags &= ~TSF_DIRTYLINE;
if ((flags & TSF_NEWLINES) != 0) if ((flags & TSF_NEWLINES) != 0) {
break; break;
} }
} while (isJSSpace(c) || c == '\n'); } else if (!isJSSpace(c)) {
if (c != '-') {
if (c == EOF_CHAR)
return EOF;
if (c != '-' && c != '\n')
flags |= TSF_DIRTYLINE; flags |= TSF_DIRTYLINE;
}
break;
}
}
// identifier/keyword/instanceof? // identifier/keyword/instanceof?
// watch out for starting with a <backslash> // watch out for starting with a <backslash>
boolean identifierStart; boolean identifierStart;
boolean isUnicodeEscapeStart = false; boolean isUnicodeEscapeStart = false;
if (c == '\\') { if (c == '\\') {
c = in.read(); c = getChar();
if (c == 'u') { if (c == 'u') {
identifierStart = true; identifierStart = true;
isUnicodeEscapeStart = true; isUnicodeEscapeStart = true;
stringBufferTop = 0; stringBufferTop = 0;
} else { } else {
identifierStart = false; identifierStart = false;
ungetChar(c);
c = '\\'; c = '\\';
in.unread();
} }
} else { } else {
identifierStart = Character.isJavaIdentifierStart((char)c); identifierStart = Character.isJavaIdentifierStart((char)c);
@ -797,7 +761,7 @@ public class TokenStream {
// an error here. // an error here.
int escapeVal = 0; int escapeVal = 0;
for (int i = 0; i != 4; ++i) { for (int i = 0; i != 4; ++i) {
c = in.read(); c = getChar();
escapeVal = (escapeVal << 4) | xDigitToInt(c); escapeVal = (escapeVal << 4) | xDigitToInt(c);
// Next check takes care about c < 0 and bad escape // Next check takes care about c < 0 and bad escape
if (escapeVal < 0) { break; } if (escapeVal < 0) { break; }
@ -809,9 +773,9 @@ public class TokenStream {
addToString(escapeVal); addToString(escapeVal);
isUnicodeEscapeStart = false; isUnicodeEscapeStart = false;
} else { } else {
c = in.read(); c = getChar();
if (c == '\\') { if (c == '\\') {
c = in.read(); c = getChar();
if (c == 'u') { if (c == 'u') {
isUnicodeEscapeStart = true; isUnicodeEscapeStart = true;
containsEscape = true; containsEscape = true;
@ -820,14 +784,16 @@ public class TokenStream {
return ERROR; return ERROR;
} }
} else { } else {
if (!Character.isJavaIdentifierPart((char)c)) { if (c == EOF_CHAR
|| !Character.isJavaIdentifierPart((char)c))
{
break; break;
} }
addToString(c); addToString(c);
} }
} }
} }
in.unread(); ungetChar(c);
String str = getStringFromBuffer(); String str = getStringFromBuffer();
if (!containsEscape) { if (!containsEscape) {
@ -854,21 +820,21 @@ public class TokenStream {
} }
} }
} }
this.string = str; this.string = (String)allStrings.intern(str);
return NAME; return NAME;
} }
// is it a number? // is it a number?
if (isDigit(c) || (c == '.' && isDigit(in.peek()))) { if (isDigit(c) || (c == '.' && isDigit(peekChar()))) {
stringBufferTop = 0; stringBufferTop = 0;
int base = 10; int base = 10;
if (c == '0') { if (c == '0') {
c = in.read(); c = getChar();
if (c == 'x' || c == 'X') { if (c == 'x' || c == 'X') {
base = 16; base = 16;
c = in.read(); c = getChar();
} else if (isDigit(c)) { } else if (isDigit(c)) {
base = 8; base = 8;
} else { } else {
@ -879,7 +845,7 @@ public class TokenStream {
if (base == 16) { if (base == 16) {
while (0 <= xDigitToInt(c)) { while (0 <= xDigitToInt(c)) {
addToString(c); addToString(c);
c = in.read(); c = getChar();
} }
} else { } else {
while ('0' <= c && c <= '9') { while ('0' <= c && c <= '9') {
@ -895,7 +861,7 @@ public class TokenStream {
base = 10; base = 10;
} }
addToString(c); addToString(c);
c = in.read(); c = getChar();
} }
} }
@ -906,15 +872,15 @@ public class TokenStream {
if (c == '.') { if (c == '.') {
do { do {
addToString(c); addToString(c);
c = in.read(); c = getChar();
} while (isDigit(c)); } while (isDigit(c));
} }
if (c == 'e' || c == 'E') { if (c == 'e' || c == 'E') {
addToString(c); addToString(c);
c = in.read(); c = getChar();
if (c == '+' || c == '-') { if (c == '+' || c == '-') {
addToString(c); addToString(c);
c = in.read(); c = getChar();
} }
if (!isDigit(c)) { if (!isDigit(c)) {
reportSyntaxError("msg.missing.exponent", null); reportSyntaxError("msg.missing.exponent", null);
@ -922,11 +888,11 @@ public class TokenStream {
} }
do { do {
addToString(c); addToString(c);
c = in.read(); c = getChar();
} while (isDigit(c)); } while (isDigit(c));
} }
} }
in.unread(); ungetChar(c);
String numString = getStringFromBuffer(); String numString = getStringFromBuffer();
double dval; double dval;
@ -959,10 +925,10 @@ public class TokenStream {
int val = 0; int val = 0;
stringBufferTop = 0; stringBufferTop = 0;
c = in.read(); c = getChar();
strLoop: while (c != quoteChar) { strLoop: while (c != quoteChar) {
if (c == '\n' || c == EOF_CHAR) { if (c == '\n' || c == EOF_CHAR) {
in.unread(); ungetChar(c);
reportSyntaxError("msg.unterminated.string.lit", null); reportSyntaxError("msg.unterminated.string.lit", null);
return ERROR; return ERROR;
} }
@ -970,7 +936,7 @@ public class TokenStream {
if (c == '\\') { if (c == '\\') {
// We've hit an escaped character // We've hit an escaped character
c = in.read(); c = getChar();
switch (c) { switch (c) {
case 'b': c = '\b'; break; case 'b': c = '\b'; break;
case 'f': c = '\f'; break; case 'f': c = '\f'; break;
@ -992,7 +958,7 @@ public class TokenStream {
addToString('u'); addToString('u');
int escapeVal = 0; int escapeVal = 0;
for (int i = 0; i != 4; ++i) { for (int i = 0; i != 4; ++i) {
c = in.read(); c = getChar();
escapeVal = (escapeVal << 4) | xDigitToInt(c); escapeVal = (escapeVal << 4) | xDigitToInt(c);
if (escapeVal < 0) { if (escapeVal < 0) {
continue strLoop; continue strLoop;
@ -1009,14 +975,14 @@ public class TokenStream {
/* Get 2 hex digits, defaulting to 'x' + literal /* Get 2 hex digits, defaulting to 'x' + literal
* sequence, as above. * sequence, as above.
*/ */
c = in.read(); c = getChar();
int escapeVal = xDigitToInt(c); int escapeVal = xDigitToInt(c);
if (escapeVal < 0) { if (escapeVal < 0) {
addToString('x'); addToString('x');
continue strLoop; continue strLoop;
} else { } else {
int c1 = c; int c1 = c;
c = in.read(); c = getChar();
escapeVal = (escapeVal << 4) | xDigitToInt(c); escapeVal = (escapeVal << 4) | xDigitToInt(c);
if (escapeVal < 0) { if (escapeVal < 0) {
addToString('x'); addToString('x');
@ -1031,27 +997,28 @@ public class TokenStream {
default: if ('0' <= c && c < '8') { default: if ('0' <= c && c < '8') {
val = c - '0'; val = c - '0';
c = in.read(); c = getChar();
if ('0' <= c && c < '8') { if ('0' <= c && c < '8') {
val = 8 * val + c - '0'; val = 8 * val + c - '0';
c = in.read(); c = getChar();
if ('0' <= c && c < '8' && val <= 037) { if ('0' <= c && c < '8' && val <= 037) {
// c is 3rd char of octal sequence only if // c is 3rd char of octal sequence only if
// the resulting val <= 0377 // the resulting val <= 0377
val = 8 * val + c - '0'; val = 8 * val + c - '0';
c = in.read(); c = getChar();
} }
} }
in.unread(); ungetChar(c);
c = val; c = val;
} }
} }
} }
addToString(c); addToString(c);
c = in.read(); c = getChar();
} }
this.string = getStringFromBuffer(); String str = getStringFromBuffer();
this.string = (String)allStrings.intern(str);
return STRING; return STRING;
} }
@ -1071,9 +1038,9 @@ public class TokenStream {
case '.': return DOT; case '.': return DOT;
case '|': case '|':
if (in.match('|')) { if (matchChar('|')) {
return OR; return OR;
} else if (in.match('=')) { } else if (matchChar('=')) {
this.op = BITOR; this.op = BITOR;
return ASSIGN; return ASSIGN;
} else { } else {
@ -1081,7 +1048,7 @@ public class TokenStream {
} }
case '^': case '^':
if (in.match('=')) { if (matchChar('=')) {
this.op = BITXOR; this.op = BITXOR;
return ASSIGN; return ASSIGN;
} else { } else {
@ -1089,9 +1056,9 @@ public class TokenStream {
} }
case '&': case '&':
if (in.match('&')) { if (matchChar('&')) {
return AND; return AND;
} else if (in.match('=')) { } else if (matchChar('=')) {
this.op = BITAND; this.op = BITAND;
return ASSIGN; return ASSIGN;
} else { } else {
@ -1099,8 +1066,8 @@ public class TokenStream {
} }
case '=': case '=':
if (in.match('=')) { if (matchChar('=')) {
if (in.match('=')) if (matchChar('='))
this.op = SHEQ; this.op = SHEQ;
else else
this.op = EQ; this.op = EQ;
@ -1111,8 +1078,8 @@ public class TokenStream {
} }
case '!': case '!':
if (in.match('=')) { if (matchChar('=')) {
if (in.match('=')) if (matchChar('='))
this.op = SHNE; this.op = SHNE;
else else
this.op = NE; this.op = NE;
@ -1124,18 +1091,18 @@ public class TokenStream {
case '<': case '<':
/* NB:treat HTML begin-comment as comment-till-eol */ /* NB:treat HTML begin-comment as comment-till-eol */
if (in.match('!')) { if (matchChar('!')) {
if (in.match('-')) { if (matchChar('-')) {
if (in.match('-')) { if (matchChar('-')) {
skipLine(); skipLine();
return getToken(); // in place of 'goto retry' return getToken(); // in place of 'goto retry'
} }
in.unread(); ungetChar('-');
} }
in.unread(); ungetChar('!');
} }
if (in.match('<')) { if (matchChar('<')) {
if (in.match('=')) { if (matchChar('=')) {
this.op = LSH; this.op = LSH;
return ASSIGN; return ASSIGN;
} else { } else {
@ -1143,7 +1110,7 @@ public class TokenStream {
return SHOP; return SHOP;
} }
} else { } else {
if (in.match('=')) { if (matchChar('=')) {
this.op = LE; this.op = LE;
return RELOP; return RELOP;
} else { } else {
@ -1153,9 +1120,9 @@ public class TokenStream {
} }
case '>': case '>':
if (in.match('>')) { if (matchChar('>')) {
if (in.match('>')) { if (matchChar('>')) {
if (in.match('=')) { if (matchChar('=')) {
this.op = URSH; this.op = URSH;
return ASSIGN; return ASSIGN;
} else { } else {
@ -1163,7 +1130,7 @@ public class TokenStream {
return SHOP; return SHOP;
} }
} else { } else {
if (in.match('=')) { if (matchChar('=')) {
this.op = RSH; this.op = RSH;
return ASSIGN; return ASSIGN;
} else { } else {
@ -1172,7 +1139,7 @@ public class TokenStream {
} }
} }
} else { } else {
if (in.match('=')) { if (matchChar('=')) {
this.op = GE; this.op = GE;
return RELOP; return RELOP;
} else { } else {
@ -1182,7 +1149,7 @@ public class TokenStream {
} }
case '*': case '*':
if (in.match('=')) { if (matchChar('=')) {
this.op = MUL; this.op = MUL;
return ASSIGN; return ASSIGN;
} else { } else {
@ -1191,13 +1158,13 @@ public class TokenStream {
case '/': case '/':
// is it a // comment? // is it a // comment?
if (in.match('/')) { if (matchChar('/')) {
skipLine(); skipLine();
return getToken(); return getToken();
} }
if (in.match('*')) { if (matchChar('*')) {
while ((c = in.read()) != -1 && while ((c = getChar()) != -1 &&
!(c == '*' && in.match('/'))) { !(c == '*' && matchChar('/'))) {
; // empty loop body ; // empty loop body
} }
if (c == EOF_CHAR) { if (c == EOF_CHAR) {
@ -1210,15 +1177,15 @@ public class TokenStream {
// is it a regexp? // is it a regexp?
if ((flags & TSF_REGEXP) != 0) { if ((flags & TSF_REGEXP) != 0) {
stringBufferTop = 0; stringBufferTop = 0;
while ((c = in.read()) != '/') { while ((c = getChar()) != '/') {
if (c == '\n' || c == EOF_CHAR) { if (c == '\n' || c == EOF_CHAR) {
in.unread(); ungetChar(c);
reportSyntaxError("msg.unterminated.re.lit", null); reportSyntaxError("msg.unterminated.re.lit", null);
return ERROR; return ERROR;
} }
if (c == '\\') { if (c == '\\') {
addToString(c); addToString(c);
c = in.read(); c = getChar();
} }
addToString(c); addToString(c);
@ -1226,17 +1193,17 @@ public class TokenStream {
int reEnd = stringBufferTop; int reEnd = stringBufferTop;
while (true) { while (true) {
if (in.match('g')) if (matchChar('g'))
addToString('g'); addToString('g');
else if (in.match('i')) else if (matchChar('i'))
addToString('i'); addToString('i');
else if (in.match('m')) else if (matchChar('m'))
addToString('m'); addToString('m');
else else
break; break;
} }
if (isAlpha(in.peek())) { if (isAlpha(peekChar())) {
reportSyntaxError("msg.invalid.re.flag", null); reportSyntaxError("msg.invalid.re.flag", null);
return ERROR; return ERROR;
} }
@ -1248,7 +1215,7 @@ public class TokenStream {
} }
if (in.match('=')) { if (matchChar('=')) {
this.op = DIV; this.op = DIV;
return ASSIGN; return ASSIGN;
} else { } else {
@ -1257,7 +1224,7 @@ public class TokenStream {
case '%': case '%':
this.op = MOD; this.op = MOD;
if (in.match('=')) { if (matchChar('=')) {
return ASSIGN; return ASSIGN;
} else { } else {
return MOD; return MOD;
@ -1268,24 +1235,24 @@ public class TokenStream {
return UNARYOP; return UNARYOP;
case '+': case '+':
if (in.match('=')) { if (matchChar('=')) {
this.op = ADD; this.op = ADD;
return ASSIGN; return ASSIGN;
} else if (in.match('+')) { } else if (matchChar('+')) {
return INC; return INC;
} else { } else {
return ADD; return ADD;
} }
case '-': case '-':
if (in.match('=')) { if (matchChar('=')) {
this.op = SUB; this.op = SUB;
c = ASSIGN; c = ASSIGN;
} else if (in.match('-')) { } else if (matchChar('-')) {
if (0 == (flags & TSF_DIRTYLINE)) { if (0 == (flags & TSF_DIRTYLINE)) {
// treat HTML end-comment after possible whitespace // treat HTML end-comment after possible whitespace
// after line start as comment-utill-eol // after line start as comment-utill-eol
if (in.match('>')) { if (matchChar('>')) {
skipLine(); skipLine();
return getToken(); return getToken();
} }
@ -1303,6 +1270,53 @@ public class TokenStream {
} }
} }
private static boolean isAlpha(int c) {
// Use 'Z' < 'a'
if (c <= 'Z') {
return 'A' <= c;
} else {
return 'a' <= c && c <= 'z';
}
}
static boolean isDigit(int c) {
return '0' <= c && c <= '9';
}
static int xDigitToInt(int c) {
// Use 0..9 < A..Z < a..z
if (c <= '9') {
c -= '0';
if (0 <= c) { return c; }
} else if (c <= 'F') {
if ('A' <= c) { return c - ('A' - 10); }
} else if (c <= 'f') {
if ('a' <= c) { return c - ('a' - 10); }
}
return -1;
}
/* As defined in ECMA. jsscan.c uses C isspace() (which allows
* \v, I think.) note that code in getChar() implicitly accepts
* '\r' == \u000D as well.
*/
public static boolean isJSSpace(int c) {
if (c <= 127) {
return c == 0x20 || c == 0x9 || c == 0xC || c == 0xB;
} else {
return c == 0xA0
|| Character.getType((char)c) == Character.SPACE_SEPARATOR;
}
}
public static boolean isJSLineTerminator(int c) {
return c == '\n' || c == '\r' || c == 0x2028 || c == 0x2029;
}
private static boolean isJSFormatChar(int c) {
return c > 127 && Character.getType((char)c) == Character.FORMAT;
}
private String getStringFromBuffer() { private String getStringFromBuffer() {
return new String(stringBuffer, 0, stringBufferTop); return new String(stringBuffer, 0, stringBufferTop);
} }
@ -1336,19 +1350,161 @@ public class TokenStream {
getLineno(), getLine(), getOffset()); getLineno(), getLine(), getOffset());
} }
public String getSourceName() { return sourceName; } private void ungetChar(int c) {
public int getLineno() { return in.getLineno(); } // can not unread past across line boundary
public int getOp() { return op; } if (ungetCursor != 0 && ungetBuffer[ungetCursor - 1] == '\n')
public String getString() { return string; } Context.codeBug();
public double getNumber() { return number; } ungetBuffer[ungetCursor++] = c;
public String getLine() { return in.getLine(); } }
public int getOffset() { return in.getOffset(); }
public int getTokenno() { return tokenno; }
public boolean eof() { return in.eof(); }
// instance variables private boolean matchChar(int test) throws IOException {
private LineBuffer in; int c = getChar();
if (c == test) {
return true;
} else {
ungetChar(c);
return false;
}
}
private int peekChar() throws IOException {
int c = getChar();
ungetChar(c);
return c;
}
private int getChar() throws IOException {
if (ungetCursor != 0) {
return ungetBuffer[--ungetCursor];
}
for(;;) {
int c;
if (sourceString != null) {
if (sourceCursor == sourceEnd) {
hitEOF = true;
return EOF_CHAR;
}
c = sourceString.charAt(sourceCursor++);
} else {
if (sourceCursor == sourceEnd) {
if (!fillSourceBuffer()) {
hitEOF = true;
return EOF_CHAR;
}
}
c = sourceBuffer[sourceCursor++];
}
if (lineEndChar >= 0) {
if (lineEndChar == '\r' && c == '\n') {
lineEndChar = '\n';
continue;
}
lineEndChar = -1;
lineStart = sourceCursor - 1;
lineno++;
}
if (c <= 127) {
if (c == '\n' || c == '\r') {
lineEndChar = c;
c = '\n';
}
} else {
if (isJSFormatChar(c)) {
continue;
}
if ((c & EOL_HINT_MASK) == 0 && isJSLineTerminator(c)) {
lineEndChar = c;
c = '\n';
}
}
return c;
}
}
private void skipLine() throws IOException {
// skip to end of line
int c;
while ((c = getChar()) != EOF_CHAR && c != '\n') { }
ungetChar(c);
}
public int getOffset() {
int n = sourceCursor - lineStart;
if (lineEndChar >= 0) { --n; }
return n;
}
public String getLine() {
if (sourceString != null) {
// String case
int lineEnd = sourceCursor;
if (lineEndChar >= 0) {
--lineEnd;
} else {
for (; lineEnd != sourceEnd; ++lineEnd) {
int c = sourceString.charAt(lineEnd);
if ((c & EOL_HINT_MASK) == 0 && isJSLineTerminator(c)) {
break;
}
}
}
return sourceString.substring(lineStart, lineEnd);
} else {
// Reader case
int lineLength = sourceCursor - lineStart;
if (lineEndChar >= 0) {
--lineLength;
} else {
// Read until the end of line
for (;; ++lineLength) {
int i = lineStart + lineLength;
if (i == sourceEnd) {
try {
if (!fillSourceBuffer()) { break; }
} catch (IOException ioe) {
// ignore it, we're already displaying an error...
break;
}
// i recalculuation as fillSourceBuffer can move saved
// line buffer and change lineStart
i = lineStart + lineLength;
}
int c = sourceBuffer[i];
if ((c & EOL_HINT_MASK) == 0 && isJSLineTerminator(c)) {
break;
}
}
}
return new String(sourceBuffer, lineStart, lineLength);
}
}
private boolean fillSourceBuffer() throws IOException {
if (sourceString != null) Context.codeBug();
if (sourceEnd == sourceBuffer.length) {
if (lineStart != 0) {
System.arraycopy(sourceBuffer, lineStart, sourceBuffer, 0,
sourceEnd - lineStart);
sourceEnd -= lineStart;
sourceCursor -= lineStart;
lineStart = 0;
} else {
char[] tmp = new char[sourceBuffer.length * 2];
System.arraycopy(sourceBuffer, 0, tmp, 0, sourceEnd);
sourceBuffer = tmp;
}
}
int n = sourceReader.read(sourceBuffer, sourceEnd,
sourceBuffer.length - sourceEnd);
if (n < 0) {
return false;
}
sourceEnd += n;
return true;
}
/* for TSF_REGEXP, etc. /* for TSF_REGEXP, etc.
* should this be manipulated by gettor/settor functions? * should this be manipulated by gettor/settor functions?
@ -1374,4 +1530,25 @@ public class TokenStream {
private char[] stringBuffer = new char[128]; private char[] stringBuffer = new char[128];
private int stringBufferTop; private int stringBufferTop;
private ObjToIntMap allStrings = new ObjToIntMap(50);
// Room to backtrace from to < on failed match of the last - in <!--
private final int[] ungetBuffer = new int[3];
private int ungetCursor;
private boolean hitEOF = false;
// Optimization for faster check for eol character: isJSLineTerminator(c)
// returns true only when (c & EOL_HINT_MASK) == 0
private static final int EOL_HINT_MASK = 0xdfd0;
private int lineStart = 0;
private int lineno;
private int lineEndChar = -1;
private String sourceString;
private Reader sourceReader;
private char[] sourceBuffer;
private int sourceEnd;
private int sourceCursor;
} }

0
js/rhino/src/org/mozilla/javascript/debug/DebugReader.java
Просмотреть файл

3
js/rhino/src/org/mozilla/javascript/resources/Messages.properties
Просмотреть файл

@ -427,9 +427,6 @@ msg.remove.sealed =\
Cannot remove a property from a sealed object. Cannot remove a property from a sealed object.
# TokenStream # TokenStream
msg.token.replaces.pushback =\
ungot token {0} replaces pushback token {1}
msg.missing.exponent =\ msg.missing.exponent =\
missing exponent missing exponent