/*
 * The contents of this file are subject to the Netscape Public
 * License Version 1.1 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of
 * the License at http://www.mozilla.org/NPL/

 * Software distributed under the License is distributed on an "AS
 * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * rights and limitations under the License.

 * The Original Code is Rhino code, released
 * May 6, 1999.

 * The Initial Developer of the Original Code is Netscape
 * Communications Corporation.  Portions created by Netscape are
 * Copyright (C) 1997-1999 Netscape Communications Corporation. All
 * Rights Reserved.

 * Contributor(s): 

 * Alternatively, the contents of this file may be used under the
 * terms of the GNU Public License (the "GPL"), in which case the
 * provisions of the GPL are applicable instead of those above.
 * If you wish to allow use of your version of this file only
 * under the terms of the GPL and not to allow others to use your
 * version of this file under the NPL, indicate your decision by
 * deleting the provisions above and replace them with the notice
 * and other provisions required by the GPL.  If you do not delete
 * the provisions above, a recipient may use your version of this
 * file under either the NPL or the GPL.
 */
/* Header file */
#include <stdio.h>
#include <ctype.h>
#include <limits.h>
#include "js-context.h"
#include "js-utils.h"
#include "js-stringbuffer.h"
#include "js-scriptruntime.h"
#include "js-runtimeexception.h"
#include "js-exceptionhandler.h"
#include "js-nativeglobal.h"
#include "js-interpreter.h"
#if defined(USE_UTF8) || defined(USE_SJIS)
# include "js-stringruntime.h"
#endif
#include "js-tokenstreamp.h"

/* Internal data */
static	JSKeyword 	keywords[] = {
    { "break",		JSTokenStream::BREAK		},
    { "case",		JSTokenStream::CASE		},
    { "continue",	JSTokenStream::CONTINUE		},
    { "default",	JSTokenStream::DEFAULT		},
    { "delete",		JSTokenStream::DELPROP		},
    { "do",		JSTokenStream::DO		},
    { "else",		JSTokenStream::ELSE		},
    {
	"export",	JSTokenStream::EXPORT	
    },
    {
 	"false",	JSTokenStream::PRIMARY | (JSTokenStream::_FALSE << 8)
    },
    { "for",		JSTokenStream::FOR		},
    { "function",	JSTokenStream::FUNCTION		},
    { "if",		JSTokenStream::IF		},
    {
	"in",		JSTokenStream::RELOP | (JSTokenStream::IN << 8)
    },
    { "new",		JSTokenStream::NEW		},
#if defined(CYGWIN)
    {
	"null",		JSTokenStream::PRIMARY | (JSTokenStream::__NULL << 8)
    },
#else /* !defined(CYGWIN) */
    {
	"null",		JSTokenStream::PRIMARY | (JSTokenStream::_NULL << 8)
    },
#endif /* !defined(CYGWIN) */
    { "return",		JSTokenStream::RETURN		},
    { "switch",		JSTokenStream::SWITCH		},
    {
	"this",		JSTokenStream::PRIMARY | (JSTokenStream::THIS << 8)
    },
    {
	"true",		JSTokenStream::PRIMARY | (JSTokenStream::_TRUE << 8)
    },
    {
	"typeof",	JSTokenStream::UNARYOP | (JSTokenStream::TYPEOF << 8)
    },
    { "var",		JSTokenStream::VAR		},
    {
	"void",		JSTokenStream::UNARYOP | (JSTokenStream::VOID << 8)
    },
    { "while",		JSTokenStream::WHILE		},
    { "with",		JSTokenStream::WITH		},

    // the following are #ifdef RESERVE_JAVA_KEYWORDS in jsscan.c
    { "abstract",	JSTokenStream::RESERVED		},
    { "boolean",	JSTokenStream::RESERVED		},
    { "byte",		JSTokenStream::RESERVED		},
    { "catch",		JSTokenStream::CATCH		},
    { "char",		JSTokenStream::RESERVED		},
    { "class",		JSTokenStream::RESERVED		},
    { "const",		JSTokenStream::RESERVED		},
    { "debugger",	JSTokenStream::RESERVED		},
    { "double",		JSTokenStream::RESERVED		},
    { "enum",		JSTokenStream::RESERVED		},
    { "extends",	JSTokenStream::RESERVED		},
    { "final",		JSTokenStream::RESERVED		},
    { "finally",	JSTokenStream::FINALLY		},
    { "float",		JSTokenStream::RESERVED		},
    { "goto",		JSTokenStream::RESERVED		},
    { "implements",	JSTokenStream::RESERVED		},
#ifdef VX
    { "import",		JSTokenStream::_IMPORT		},
#else
    { "import",		JSTokenStream::IMPORT		},
#endif
    {
	"instanceof",	JSTokenStream::RELOP | (JSTokenStream::INSTANCEOF << 8)
    },
    { "int",		JSTokenStream::RESERVED		},
    { "interface",	JSTokenStream::RESERVED		},
    { "long",		JSTokenStream::RESERVED		},
    { "native",		JSTokenStream::RESERVED		},
    { "package",	JSTokenStream::RESERVED		},
    { "private",	JSTokenStream::RESERVED		},
    { "protected",	JSTokenStream::RESERVED		},
    { "public",		JSTokenStream::RESERVED		},
    { "short",		JSTokenStream::RESERVED		},
    { "static",		JSTokenStream::RESERVED		},
    { "super",		JSTokenStream::RESERVED		},
    { "synchronized",	JSTokenStream::RESERVED		},
    { "throw",		JSTokenStream::THROW		},
    { "throws",		JSTokenStream::RESERVED		},
    { "transient",	JSTokenStream::RESERVED		},
    { "try",		JSTokenStream::TRY		},
    { "volatile",	JSTokenStream::RESERVED		},
} ;
#ifdef VX
static	JSKeywordHashtable	*keywordHashtable;
#endif

/* Class data */
jsString_t	*JSTokenStream::names;	
int		JSTokenStream::namesLength;	

/* Procedures */
void
JSTokenStream::checkNames()
{
#ifdef STATS
    if (names == NULL) {
#else
    if ((JSContext::printTrees || JSInterpreter::printICode)
	&& names == NULL) {
#endif
	static jsString_t a[] = {
	    "error",
	    "eof",
	    "eol",
	    "popv",
	    "enterwith",
	    "leavewith",
	    "return",
	    "goto",
	    "ifeq",
	    "ifne",
	    "dup",
	    "setname",
	    "bitor",
	    "bitxor",
	    "bitand",
	    "eq",
	    "ne",
	    "lt",
	    "le",
	    "gt",
	    "ge",
	    "lsh",
	    "rsh",
	    "ursh",
	    "add",
	    "sub",
	    "mul",
	    "div",
	    "mod",
	    "bitnot",
	    "neg",
	    "new",
	    "delprop",
	    "typeof",
	    "nameinc",
	    "propinc",
	    "eleminc",
	    "namedec",
	    "propdec",
	    "elemdec",
	    "getprop",
	    "setprop",
	    "getelem",
	    "setelem",
	    "call",
	    "name",
	    "number",
	    "string",
	    "zero",
	    "one",
	    "null",
	    "this",
	    "false",
	    "true",
	    "sheq",
	    "shne",
	    "closure",
	    "object",
	    "pop",
	    "pos",
	    "varinc",
	    "vardec",
	    "bindname",
	    "throw",
	    "in",
	    "instanceof",
	    "gosub",
	    "retsub",
	    "callspecial",
	    "getthis",
	    "newtemp",
	    "usetemp",
	    "getbase",
	    "getvar",
	    "setvar",
	    "undefined",
	    "try",
	    "endtry",
	    "newscope",
	    "typeofname",
	    "enuminit",
	    "enumnext",
	    "getproto",
	    "getparent",
	    "setproto",
	    "setparent",
	    "scope",
	    "getscopeparent",
	    "jthrow",
	    "semi",
	    "lb",
	    "rb",
	    "lc",
	    "rc",
	    "lp",
	    "rp",
	    "comma",
	    "assign",
	    "hook",
	    "colon",
	    "or",
	    "and",
	    "eqop",
	    "relop",
	    "shop",
	    "unaryop",
	    "inc",
	    "dec",
	    "dot",
	    "primary",
	    "function",
	    "export",
	    "import",
	    "if",
	    "else",
	    "switch",
	    "case",
	    "default",
	    "while",
	    "do",
	    "for",
	    "break",
	    "continue",
	    "var",
	    "with",
	    "catch",
	    "finally",
	    "reserved",
	    "nop",
	    "not",
	    "pre",
	    "post",
	    "void",
	    "block",
	    "arraylit",
	    "objlit",
	    "label",
	    "target",
	    "loop",
	    "enumdone",
	    "exprstmt",
	    "parent",
	    "convert",
	    "jsr",
	    "newlocal",
	    "uselocal",
	    "script",
	    "line",
	    "sourcefile",
	} ;
	names = a;
	namesLength = TableSize(names);
    }
}
jsBool_t
JSTokenStream::isAlpha(int c)
{
#ifdef FASTER
    return (isalpha(c) != 0);
#else
    return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'));
#endif
}
jsBool_t
JSTokenStream::isDigit(int c)
{
#ifdef FASTER
    return (isdigit(c) != 0);
#else
    return (c >= '0' && c <= '9');
#endif
}
jsBool_t
JSTokenStream::isXDigit(int c)
{
#ifdef FASTER
    return (isxdigit(c) != 0);
#else
    return ((c >= '0' && c <= '9')
	    || (c >= 'a' && c <= 'f')
	    || (c >= 'A' && c <= 'F'));
#endif
}
jsBool_t
JSTokenStream::isJSIdentifier(jsString_t s)
{
    int length = strlen(s);

    if (length == 0 || !isJavaIdentifierStart(s[0])) {
	return FALSE;
    }

    for (int i = 1; i < length; i++) {
	jsChar_t c = s[i];

	if (!isJavaIdentifierPart(c)) {
	    if (c == '\\') {
		if (! ((i + 5) < length)
		    && (s[i + 1] == 'u')
		    && isXDigit(s[i + 2])
		    && isXDigit(s[i + 3])
		    && isXDigit(s[i + 4])
		    && isXDigit(s[i + 5])) {
		    return FALSE;
		}
	   }
	}
    }

    return TRUE;
}
int
JSTokenStream::digit(jsChar_t c, int radix)
{
    if (c >= '0' && c <= '9') {
	return c - '0';
    }
    if (radix == 16) {
	c = tolower(c);
	if (c >= 'a' && c <= 'f') {
	    return c - 'a' + 10;
	}
    }
    return 0;
}
jsBool_t
JSTokenStream::isJavaIdentifierPart(int c)
{
    return isJavaIdentifierStart(c) || isDigit(c);
}
jsBool_t
JSTokenStream::isJavaIdentifierStart(int c)
{
    return isAlpha(c) || (c=='$') || (c=='_');
}
void
JSTokenStream::setString(JSContext *cx, jsString_t string)
{
#ifndef FASTER
    if (this->string != NULL) JS_DeleteZoneObj(cx, this->string);
#endif
    this->string = string;
}
jsString_t
JSTokenStream::tokenToString(JSContext *cx, int token)
{
    if (JSContext::printTrees) {
	jsString_t string;
	int length;

	checkNames();
	if (token + 1 >= namesLength) {
	    return NULL;
	}

	if (token == UNARYOP ||
	    token == ASSIGN ||
	    token == PRIMARY ||
	    token == EQOP ||
	    token == SHOP ||
	    token == RELOP) {
	    length = strlen(names[token + 1]) + TableSize(" ") - 1
			+ strlen(names[this->op + 1]) + 1;
	    string = (jsString_t) JS_NewZoneObj(cx, sizeof (*string) * length);

	    sprintf(string, "%s %s", names[token + 1], names[this->op + 1]);
	    return string;
	}

	if (token == STRING || token == OBJECT || token == NAME) {
	    length = strlen(names[token + 1]) 
			+ strlen(this->string) + TableSize(" ` '") - 1 + 1;
	    string = (jsString_t) JS_NewZoneObj(cx, sizeof (*string) * length);
	    sprintf(string, "%s `%s '", names[token + 1], this->string);
	    return string;
	}

	if (token == NUMBER) {
	    length = TableSize("NUMBER ") - 1 + 32;
	    string = (jsString_t) JS_NewZoneObj(cx, sizeof (*string) * length);
	    sprintf(string, "NUMBER %f", this->number);
	    return string;
	}

	return JS_NewZoneString(cx, names[token + 1]);
    }
    return JS_NewZoneString(cx, "");
}
jsString_t
JSTokenStream::tokenToName(int type)
{
    checkNames();
    return names == NULL ? (jsString_t) "" : names[type + 1];
}
jsBool_t
JSTokenStream::isJSSpace(int c)
{
    const int BOM = 0xfeff;

    return (isspace(c) != 0 || ( c & 0xffff ) == BOM );
}
jsBool_t
JSTokenStream::isJSLineTerminator(int c)
{
    return (c == '\n' || c == '\r');
}

int
JSTokenStream::stringToKeyword(jsString_t name)
{
#ifdef VX
    int value = keywordHashtable->get(name);

    if (value == 0) {
	return _EOF;
    }
    this->op = value >> 8;
    return value & 0xff;
#else /* !VX */
    class bsearchFuncs {
	public:
	    static int	compare(const void *a, const void *b) {
		jsString_t string = (jsString_t) a;
		JSKeyword *keyword = (JSKeyword *) b;

		return strcmp(string, keyword->string);
	    }
    } ;

    JSKeyword *result;

    result = (JSKeyword *)
		bsearch(name, keywords, TableSize(keywords),
			sizeof (keywords[0]), &bsearchFuncs::compare);
    if (result == NULL) {
	return _EOF;
    }
    int x = result->value;

    this->op = x >> 8;
    return x & 0xff;
#endif /* VX */
}
JSScriptable *
JSTokenStream::getScope()
{ 
    return scope;
}
/* return and pop the token from the stream if it matches...
 * otherwise return null
 */
jsBool_t
JSTokenStream::matchToken(JSContext *cx, int toMatch)
{
    int token = getToken(cx);

    if (token == toMatch) {
	return TRUE;
    }
    // didn't match, push back token
    tokenno--;
    this->pushbackToken = token;
    return FALSE;
}
void
JSTokenStream::clearPushback()
{
    this->pushbackToken = _EOF;
}
void
JSTokenStream::ungetToken(JSContext *cx, int tt)
{
    if (this->pushbackToken != _EOF && tt != _ERROR) {
	jsString_t message = JSContext::getMessage(cx,
				"msg.token.replaces.pushback");
	cx->exceptionHandler->throwEx(cx, new (cx) JSRuntimeException(message));
    }
    this->pushbackToken = tt;
    tokenno--;
}
int
JSTokenStream::peekToken(JSContext *cx)
{
    int result = getToken(cx);

    this->pushbackToken = result;
    tokenno--;
    return result;
}
int
JSTokenStream::peekTokenSameLine(JSContext *cx)
{
    int result;

    flags |= TSF_NEWLINES;	  // SCAN_NEWLINES from jsscan.h
    result = peekToken(cx);
    flags &= ~TSF_NEWLINES;	 // HIDE_NEWLINES from jsscan.h
    if (this->pushbackToken == EOL) {
	this->pushbackToken = _EOF;
    }
    return result;
}
/* As defined in ECMA.  jsscan.c uses C isspace() (which allows
 * \v, I think.)  note that code in in->read() implicitly accepts
 * '\r' == \u000D as well.
 */
int
JSTokenStream::getToken(JSContext *cx)
{
    int c;

    tokenno++;

    // Check for pushed-back token
    if (this->pushbackToken != _EOF) {
	int result = this->pushbackToken;

	this->pushbackToken = _EOF;
	return result;
    }

    // Eat whitespace, possibly sensitive to newlines.
    do {
	c = in.read(cx);
	if (c == '\n') {
	    if ((flags & TSF_NEWLINES) != 0) {
		break;
	    }
	}
    } while (isJSSpace(c) || c == '\n');

    if (c == EOF_CHAR) {
	return _EOF;
    }

    // identifier/keyword/instanceof?
    // watch out for starting with a <backslash>
    jsBool_t isUnicodeEscapeStart = FALSE;

    if (c == '\\') {
	c = in.read(cx);
	if (c == 'u') {
	    isUnicodeEscapeStart = TRUE;
	} else {
	    c = '\\';
	}
	// always unread the 'u' or whatever, we need 
	// to start the string below at the <backslash>.
	in.unread();
    }
    if (isUnicodeEscapeStart || isJavaIdentifierStart((jsChar_t) c)) {
	in.startString();

	jsBool_t  containsEscape = isUnicodeEscapeStart;			
	do {
	    c = in.read(cx);
	    if (c == '\\') {
		c = in.read(cx);
		containsEscape = (c == 'u');
	    }					
	} while (isJavaIdentifierPart((jsChar_t) c));
	in.unread();

	int result;

	jsString_t str = in.getString(cx);
	// OPT we shouldn't have to make a string (object!) to
	// check if it's a keyword.
			
	// strictly speaking we should probably push-back
	// all the bad characters if the <backslash>uXXXX  
	// sequence is malformed. But since there isn't a  
	// correct context(is there?) for a bad Unicode  
	// escape sequence after an identifier, we can report
	// an error here.
	if (containsEscape) {
	    jsString_t ca = str;
	    JSStringBuffer x(cx);
	    jsString_t s = strstr(str, "\\u");
	    int escStart = s != NULL ? (s - str) : -1;
	    int start = 0;
	    int length = strlen(str);

	    while (escStart != -1) {
		x.append(ca, start, escStart);

		jsBool_t goodEscape = FALSE;

		if ((escStart + 5) < length) {
		    if (isXDigit(ca[escStart + 2])) {
			int val = digit(ca[escStart + 2], 16);
			if (isXDigit(ca[escStart + 3])) {
			    val = (val << 4) | digit(ca[escStart + 3], 16);
			    if (isXDigit(ca[escStart + 4])) {
				val = (val << 4) | digit(ca[escStart + 4], 16);
				if (isXDigit(ca[escStart + 5])) {
				    val = (val << 4)
					  | digit(ca[escStart + 5], 16);
				    x.append((jsChar_t) val);
				    start = escStart + 6;
				    goodEscape = TRUE;
				}
			    }
			}
		    }
		}
		if (!goodEscape) {
		    reportSyntaxError(cx, "msg.invalid.escape");
		    return _ERROR;
		}
		escStart = (s = strstr(str + start, "\\u")) != NULL ?
				(s - str) : -1;
	    }
	    x.append(ca, start, length - start);
#ifndef FASTER
	    JS_DeleteZoneObj(cx, str);
#endif
	    str = x.offString();
	} else {
	    // Return the corresponding token if it's a keyword
	    if ((result = stringToKeyword(str)) != _EOF) {
#ifndef FASTER
		JS_DeleteZoneObj(cx, str);
#endif
	  	return result;
	    }
	}

	setString(cx, str);
	return NAME;
    }

    // is it a number?
    if (isDigit(c) || (c == '.' && isDigit(in.peek(cx)))) {
	int base = 10;
	in.startString();

	double dval = JSScriptRuntime::NaN;
	int64_t longval = 0;
	jsBool_t isInteger = TRUE;

	if (c == '0') {
	    c = in.read(cx);
	    if (c == 'x' || c == 'X') {
		c = in.read(cx);
		base = 16;
		// restart the string, losing leading 0x
		in.startString();
	    } else if (isDigit(c)) {
		if (c < '8') {
		    base = 8;
		    // Restart the string, losing the leading 0
		    in.startString();
		} else {
		    /* Checking against c < '8' is non-ECMA, but
		     * is required to support legacy code; we've
		     * supported it in the past, and it's likely
		     * that "08" and "09" are in use in code
		     * having to do with dates.  So we need to
		     * support it, which makes our behavior a
		     * superset of ECMA in this area.  We raise a
		     * warning if a non-octal digit is
		     * encountered, then proceed as if it were
		     * decimal.
		     */
		    JSContext::reportWarning(cx,
				JSContext::getMessage(cx,
						"msg.bad.octal.literal %c", c),
			 	getSourceName(), in.getLineno(),
			 	getLine(cx), getOffset());
		    // implicitly retain the leading 0
		}
	    } else {
		// implicitly retain the leading 0
	    }
	}

	while (isXDigit(c)) {
	    if (base < 16 && (isAlpha(c) || (base == 8 && c >= '8'))) {
		break;
	    }
	    c = in.read(cx);
	}

	if (base == 10 && (c == '.' || c == 'e' || c == 'E')) {
	    isInteger = FALSE;
	    if (c == '.') {
		do {
		    c = in.read(cx);
		} while (isDigit(c));
	    }

	    if (c == 'e' || c == 'E') {
		c = in.read(cx);
		if (c == '+' || c == '-') {
		    c = in.read(cx);
		}
		if (!isDigit(c)) {
		    in.getString(cx); // throw away string in progress
		    reportSyntaxError(cx, "msg.missing.exponent");
		    return _ERROR;
		}
		do {
		    c = in.read(cx);
		} while (isDigit(c));
	    }
	}
	in.unread();
	jsString_t numString = in.getString(cx);

	if (base == 10 && !isInteger) {
	    if (sscanf(numString, "%lf", &dval) != 1) {
		reportSyntaxError(cx, "msg.caught.nfe %s", numString);
		return _ERROR;
	    }
	} else {
	    dval = JSScriptRuntime::stringToNumber(numString, 0, base);
	    longval = (int64_t) dval;

	    // is it an integral fits-in-a-long value?
	    if (longval != dval) {
		isInteger = FALSE;
	    }
	}

	if (!isInteger) {
	    /* Can't handle floats right now, because postfix INC/DEC
	       generate Doubles, but I would generate a Float through this
	       path, and it causes a stack mismatch. FIXME (MS)
	       if (Float.MIN_VALUE <= dval && dval <= Float.MAX_VALUE)
	       this->number = new Float((float) dval);
	       else
	    */
	    this->number = (double) dval;
	} else {
	    // We generate the smallest possible type here
	    if (SCHAR_MIN <= longval && longval <= SCHAR_MAX) {
		this->number = (int8_t) longval;
	    } else if (SHRT_MIN <= longval && longval <= SHRT_MAX) {
		this->number = (short) longval;
	    } else if (INT_MIN <= longval && longval <= INT_MAX) {
		this->number = (int) longval;
	    } else {
		// May lose some precision here, but that's the 
		// appropriate semantics.
		this->number = (double) longval;
	    }
	}
#ifndef FASTER
	JS_DeleteZoneObj(cx, numString);
#endif
	return NUMBER;
    }

    // is it a string?
    if (c == '"' || c == '\'') {
	// We attempt to accumulate a string the fast way, by
	// building it directly out of the reader.  But if there
	// are any escaped characters in the string, we revert to
	// building it out of a StringBuffer.

	JSStringBuffer stringBuf(cx, 0);
	jsBool_t used = FALSE;

	int quoteChar = c;
	int val = 0;

	c = in.read(cx);
	in.startString(); // start after the first "
	while(c != quoteChar) {
	    if (c == '\n' || c == EOF_CHAR) {
		in.unread();
		in.getString(cx); // throw away the string in progress
		reportSyntaxError(cx, "msg.unterminated.string.lit", NULL);
		return _ERROR;
	    }

	    if (c == '\\') {
		// We've hit an escaped character; revert to the
		// slow method of building a string.
		// Don't include the backslash
		if (!used) {
		    in.unread();

		    jsString_t s = in.getString(cx);

		    stringBuf.append(s);
#ifndef FASTER
		    JS_DeleteZoneObj(cx, s);
#endif
		    in.read(cx);
		    used = TRUE;
		}

		switch (c = in.read(cx)) {
		case 'b': c = '\b'; break;
		case 'f': c = '\f'; break;
		case 'n': c = '\n'; break;
		case 'r': c = '\r'; break;
		case 't': c = '\t'; break;
		case 'v': c = 0x0b; break;
		// case 'v': c = '\u000B'; break;
		    // \v a late addition to the ECMA spec.
		    // '\v' doesn't seem to be valid Java.

		default:
		    if (isDigit(c) && c < '8') {
			val = c - '0';
			c = in.read(cx);
			if (isDigit(c) && c < '8') {
			    val = 8 * val + c - '0';
			    c = in.read(cx);
			    if (isDigit(c) && c < '8') {
				val = 8 * val + c - '0';
				c = in.read(cx);
			    }
			}
			in.unread();
			if (val > 0377) {
			    reportSyntaxError(cx, "msg.oct.esc.too.large");
			    return _ERROR;
			}
			c = val;
		    } else if (c == 'u') {
			/*
			 * Get 4 hex digits; if the u escape is not
			 * followed by 4 hex digits, use 'u' + the literal
			 * character sequence that follows.  Do some manual
			 * match (OK because we're in a string) to avoid
			 * multi-char match on the underlying stream.
			 */
			int c1, c2, c3, c4;

			c1 = in.read(cx);
			if (!isXDigit(c1)) {
			    in.unread();
			    c = 'u';
			} else {
			    val = digit((jsChar_t) c1, 16);
			    c2 = in.read(cx);
			    if (!isXDigit(c2)) {
				in.unread();
				stringBuf.append('u');
				c = c1;
			    } else {
				val = 16 * val + digit((jsChar_t) c2, 16);
				c3 = in.read(cx);
				if (!isXDigit(c3)) {
				    in.unread();
				    stringBuf.append('u');
				    stringBuf.append((jsChar_t) c1);
				    c = c2;
				} else {
				    val = 16 * val + digit((jsChar_t) c3, 16);
				    c4 = in.read(cx);
				    if (!isXDigit(c4)) {
					in.unread();
					stringBuf.append('u');
					stringBuf.append((jsChar_t) c1);
					stringBuf.append((jsChar_t) c2);
					c = c3;
				    } else {
					// got 4 hex digits! Woo Hoo!
					val = 16 * val + digit((jsChar_t) c4,
								16);
					c = val;
				    }
				}
			    }
			}
		    } else if (c == 'x') {
			/* Get 2 hex digits, defaulting to 'x' + literal
			 * sequence, as above.
			 */
			int c1, c2;

			c1 = in.read(cx);
			if (!isXDigit(c1)) {
			    in.unread();
			    c = 'x';
			} else {
			    val = digit((jsChar_t) c1, 16);
			    c2 = in.read(cx);
			    if (!isXDigit(c2)) {
				in.unread();
				stringBuf.append('x');
				c = c1;
			    } else {
				// got 2 hex digits
				val = 16 * val
				    + digit((jsChar_t) c2, 16);
				c = val;
			    }
			}
		    }
		}
	    }

	    if (used) {
#if defined(USE_UTF8) || defined(USE_SJIS)
		if (c < 0x80) {
		    stringBuf.append( (jsChar_t) c);
		} else {
		    jsChar_t mb[JSStringRuntime::_MB_CUR_MAX];
		    int n;

		    n = JSStringRuntime::wctomb(mb, c);
		    stringBuf.append(mb, 0, n);
		}
#else /* !(defined(USE_UTF8) || defined(USE_SJIS)) */
		stringBuf.append((jsChar_t) c);
#endif /* defined(USE_UTF8) || defined(USE_SJIS) */
	    }
	    c = in.read(cx);
	}

	if (used) {
#ifdef FASTER
	    if (this->_string != NULL) {
		JS_DeleteZoneObj(cx, this->_string);
	    }
	    this->_string =  stringBuf.offString();
	    setString(cx, this->_string);
#else
	    setString(cx, stringBuf.offString());
#endif
	} else {
	    in.unread(); // miss the trailing "
	    setString(cx, in.getString(cx));
	    in.read(cx);
	}
	return STRING;
    }

    switch (c) {
    case '\n': return EOL;
    case ';': return SEMI;
    case '[': return LB;
    case ']': return RB;
    case '{': return LC;
    case '}': return RC;
    case '(': return LP;
    case ')': return RP;
    case ',': return COMMA;
    case '?': return HOOK;
    case ':': return COLON;
    case '.': return DOT;

    case '|':
	if (in.match(cx, '|')) {
	    return OR;
	} else if (in.match(cx, '=')) {
	    this->op = BITOR;
	    return ASSIGN;
	} else {
	    return BITOR;
	}

    case '^':
	if (in.match(cx, '=')) {
	    this->op = BITXOR;
	    return ASSIGN;
	} else {
	    return BITXOR;
	}

    case '&':
	if (in.match(cx, '&')) {
	    return AND;
	} else if (in.match(cx, '=')) {
	    this->op = BITAND;
	    return ASSIGN;
	} else {
	    return BITAND;
	}

    case '=':
	if (in.match(cx, '=')) {
	    if (in.match(cx, '=')) {
		this->op = SHEQ;
	    } else {
		this->op = EQ;
	    }
	    return EQOP;
	} else {
	    this->op = NOP;
	    return ASSIGN;
	}

    case '!':
	if (in.match(cx, '=')) {
	    if (in.match(cx, '=')) {
		this->op = SHNE;
	    } else {
		this->op = NE;
	    }
	    return EQOP;
	} else {
	    this->op = NOT;
	    return UNARYOP;
	}

    case '<':
	/* NB:treat HTML begin-comment as comment-till-eol */
	if (in.match(cx, '!')) {
	    if (in.match(cx, '-')) {
		if (in.match(cx, '-')) {
		    while ((c = in.read(cx)) != EOF_CHAR && c != '\n')
			/* skip to end of line */;
		    in.unread();
		    return getToken(cx);  // in place of 'goto retry'
		}
		in.unread();
	    }
	    in.unread();
	}
	if (in.match(cx, '<')) {
	    if (in.match(cx, '=')) {
		this->op = LSH;
		return ASSIGN;
	    } else {
		this->op = LSH;
		return SHOP;
	    }
	} else {
	    if (in.match(cx, '=')) {
		this->op = LE;
		return RELOP;
	    } else {
		this->op = LT;
		return RELOP;
	    }
	}

    case '>':
	if (in.match(cx, '>')) {
	    if (in.match(cx, '>')) {
		if (in.match(cx, '=')) {
		    this->op = URSH;
		    return ASSIGN;
		} else {
		    this->op = URSH;
		    return SHOP;
		}
	    } else {
		if (in.match(cx, '=')) {
		    this->op = RSH;
		    return ASSIGN;
		} else {
		    this->op = RSH;
		    return SHOP;
		}
	    }
	} else {
	    if (in.match(cx, '=')) {
		this->op = GE;
		return RELOP;
	    } else {
		this->op = GT;
		return RELOP;
	    }
	}

    case '*':
	if (in.match(cx, '=')) {
	    this->op = MUL;
	    return ASSIGN;
	} else {
	    return MUL;
	}

    case '/':
	// is it a // comment?
	if (in.match(cx, '/')) {
	    while ((c = in.read(cx)) != EOF_CHAR && c != '\n')
		/* skip to end of line */;
	    in.unread();
	    return getToken(cx);
	}
	if (in.match(cx, '*')) {
	    while ((c = in.read(cx)) != -1
		   && !(c == '*' && in.match(cx, '/'))) {
		if (c == '\n') {
		} else if (c == '/' && in.match(cx, '*')) {
		    if (in.match(cx, '/')) {
			return getToken(cx);
		    }
		    reportSyntaxError(cx, "msg.nested.comment", NULL);
		    return _ERROR;
		}
	    }
	    if (c == EOF_CHAR) {
		reportSyntaxError(cx, "msg.unterminated.comment", NULL);
		return _ERROR;
	    }
	    return getToken(cx);  // `goto retry'
	}

	// is it a regexp?
	if ((flags & TSF_REGEXP) != 0) {
	    // We don't try to use the in.startString/in.getString
	    // approach, because escaped characters (which break it)
	    // seem likely to be common.
	    JSStringBuffer re(cx);

	    while ((c = in.read(cx)) != '/') {
		if (c == '\n' || c == EOF_CHAR) {
		    in.unread();
		    reportSyntaxError(cx, "msg.unterminated.re.lit", NULL);
		    return _ERROR;
		}
		if (c == '\\') {
		    re.append((jsChar_t) c);
		    c = in.read(cx);
		}

		re.append((jsChar_t) c);
	    }

	    JSStringBuffer flagsBuf(cx);

	    while (TRUE) {
		if (in.match(cx, 'g')) {
		    flagsBuf.append('g');
		} else if (in.match(cx, 'i')) {
		    flagsBuf.append('i');
		} else if (in.match(cx, 'm')) {
		    flagsBuf.append('m');
		} else {
		    break;
		}
	    }

	    if (isAlpha(in.peek(cx))) {
		reportSyntaxError(cx, "msg.invalid.re.flag", NULL);
		return _ERROR;
	    }

#ifdef FASTER
	    if (this->regExp != NULL) {
		JS_DeleteZoneObj(cx, this->regExp);
	    }
	    this->regExp = re.offString();
	    setString(cx, this->regExp);
#else
	    setString(cx, re.offString());
#endif
	    if (this->regExpFlags != NULL) {
		JS_DeleteZoneObj(cx, this->regExpFlags);
	    }
	    this->regExpFlags = flagsBuf.offString();
	    return OBJECT;
	}


	if (in.match(cx, '=')) {
	    this->op = DIV;
	    return ASSIGN;
	} else {
	    return DIV;
	}

    case '%':
	this->op = MOD;
	if (in.match(cx, '=')) {
	    return ASSIGN;
	} else {
	    return MOD;
	}

    case '~':
	this->op = BITNOT;
	return UNARYOP;

    case '+':
    case '-':
	if (in.match(cx, '=')) {
	    if (c == '+') {
		this->op = ADD;
		return ASSIGN;
	    } else {
		this->op = SUB;
		return ASSIGN;
	    }
	} else if (in.match(cx, (jsChar_t) c)) {
	    if (c == '+') {
		return INC;
	    } else {
		return DEC;
	    }
	} else if (c == '-') {
	    return SUB;
	} else {
	    return ADD;
	}

    default:
	reportSyntaxError(cx, "msg.illegal.character", NULL);
	return _ERROR;
    }
}
void
JSTokenStream::reportSyntaxError(JSContext *cx, jsString_t messageProperty, ...)
{
    va_list ap;

    va_start(ap, messageProperty);

    jsString_t message = JSContext::getMessage(cx, messageProperty, ap);

    va_end(ap);

    if (scope != NULL) {
	// We're probably in an eval. Need to throw an exception.
	cx->exceptionHandler->throwEx(
                cx,
		JSNativeGlobal::constructError(cx, "SyntaxError",
			message, scope, getSourceName(),
			getLineno(), getOffset(), getLine(cx)));
    } else {
	JSContext::reportError(cx, message, getSourceName(),
			    getLineno(), getLine(cx), getOffset());
    }
}
void
JSTokenStream::destroy(JSContext *cx)
{
#ifdef FASTER
    if (_string != NULL) JS_DeleteZoneObj(cx, _string);
    if (regExp != NULL) JS_DeleteZoneObj(cx, regExp);
#else
    if (string != NULL) JS_DeleteZoneObj(cx, string);
#endif
    in.destroy(cx);
    JSZoneNew::destroy(cx);
}

JSTokenStream::JSTokenStream(JSContext *cx, JSReader *in, JSScriptable *scope,
		  	jsString_t sourceName, int lineno)
			: in(cx, in, lineno)
{
    this->scope = scope;
    this->pushbackToken = _EOF;
    this->sourceName = sourceName;
    flags = 0;
    string = NULL;
    tokenno = 1; //avoid purify UMR error
#ifdef FASTER
    _string = NULL;
    regExp = NULL;
#endif
    regExpFlags = NULL;
}

void
JSTokenStream::classInitialize()
{
#ifdef VX
    static JSKeywordAllocator allocator;

    keywordHashtable = new JSKeywordHashtable(&allocator);
    for (int i = 0; i < ( int ) TableSize(keywords); ++i) {
	keywordHashtable->put(keywords[i].string, keywords[i].value);
    }
#else /* !VX */
    class qsortFuncs {
	public:
	    static int	compare(const void *a, const void *b) {
		JSKeyword *kwa = (JSKeyword *) a;
		JSKeyword *kwb = (JSKeyword *) b;

		return strcmp(kwa->string, kwb->string);
	    }
    } ;

    qsort(keywords, TableSize(keywords), sizeof (keywords[0]),
	&qsortFuncs::compare);
#endif /* VX */
}
