/*
 * The contents of this file are subject to the Netscape Public
 * License Version 1.1 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of
 * the License at http://www.mozilla.org/NPL/

 * Software distributed under the License is distributed on an "AS
 * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * rights and limitations under the License.

 * The Original Code is Rhino code, released
 * May 6, 1999.

 * The Initial Developer of the Original Code is Netscape
 * Communications Corporation.  Portions created by Netscape are
 * Copyright (C) 1997-1999 Netscape Communications Corporation. All
 * Rights Reserved.

 * Contributor(s): 

 * Alternatively, the contents of this file may be used under the
 * terms of the GNU Public License (the "GPL"), in which case the
 * provisions of the GPL are applicable instead of those above.
 * If you wish to allow use of your version of this file only
 * under the terms of the GPL and not to allow others to use your
 * version of this file under the NPL, indicate your decision by
 * deleting the provisions above and replace them with the notice
 * and other provisions required by the GPL.  If you do not delete
 * the provisions above, a recipient may use your version of this
 * file under either the NPL or the GPL.
 */
/* Header file */
#include <stdio.h>
#include <ctype.h>
#include <limits.h>
#include "js-context.h"
#include "js-utils.h"
#include "js-stringbuffer.h"
#include "js-scriptruntime.h"
#include "js-runtimeexception.h"
#include "js-exceptionhandler.h"
#include "js-nativeglobal.h"
#include "js-interpreter.h"
#if defined(USE_UTF8) || defined(USE_SJIS)
# include "js-stringruntime.h"
#endif
#include "js-tokenstreamp.h"

/* Internal data */
static JSKeyword keywords[] = {
    {"break", JSTokenStream::BREAK},
    {"case", JSTokenStream::CASE},
    {"continue", JSTokenStream::CONTINUE},
    {"default", JSTokenStream::DEFAULT},
    {"delete", JSTokenStream::DELPROP},
    {"do", JSTokenStream::DO},
    {"else", JSTokenStream::ELSE},
    {
            "export", JSTokenStream::EXPORT},
    {
                "false", JSTokenStream::PRIMARY | (JSTokenStream::_FALSE << 8)
            },
    {"for", JSTokenStream::FOR},
    {"function", JSTokenStream::FUNCTION},
    {"if", JSTokenStream::IF},
    {
                "in", JSTokenStream::RELOP | (JSTokenStream::IN << 8)
            },
    {"new", JSTokenStream::NEW},
    {
                "null", JSTokenStream::PRIMARY | (JSTokenStream::_NULL << 8)
            },
    {"return", JSTokenStream::RETURN},
    {"switch", JSTokenStream::SWITCH},
    {
                "this", JSTokenStream::PRIMARY | (JSTokenStream::THIS << 8)
            },
    {
                "true", JSTokenStream::PRIMARY | (JSTokenStream::_TRUE << 8)
            },
    {
                "typeof", JSTokenStream::UNARYOP | (JSTokenStream::TYPEOF << 8)
            },
    {"var", JSTokenStream::VAR},
    {
#if defined(TARGET_OS_LINUX)
                "void", JSTokenStream::UNARYOP | (JSTokenStream::VOIDX << 8)
#else
                "void", JSTokenStream::UNARYOP | (JSTokenStream::VOID << 8)
#endif
            },
    {"while", JSTokenStream::WHILE},
    {"with", JSTokenStream::WITH},

    // the following are #ifdef RESERVE_JAVA_KEYWORDS in jsscan.c
    {"abstract", JSTokenStream::RESERVED},
    {"boolean", JSTokenStream::RESERVED},
    {"byte", JSTokenStream::RESERVED},
    {"catch", JSTokenStream::CATCH},
    {"char", JSTokenStream::RESERVED},
    {"class", JSTokenStream::RESERVED},
    {"const", JSTokenStream::RESERVED},
    {"debugger", JSTokenStream::RESERVED},
    {"double", JSTokenStream::RESERVED},
    {"enum", JSTokenStream::RESERVED},
    {"extends", JSTokenStream::RESERVED},
    {"final", JSTokenStream::RESERVED},
    {"finally", JSTokenStream::FINALLY},
    {"float", JSTokenStream::RESERVED},
    {"goto", JSTokenStream::RESERVED},
    {"implements", JSTokenStream::RESERVED},
#ifdef VX
    {"import", JSTokenStream::_IMPORT},
#else
    {"import", JSTokenStream::IMPORT},
#endif
    {
                "instanceof", JSTokenStream::RELOP | (JSTokenStream::INSTANCEOF << 8)
            },
    {"int", JSTokenStream::RESERVED},
    {"interface", JSTokenStream::RESERVED},
    {"long", JSTokenStream::RESERVED},
    {"native", JSTokenStream::RESERVED},
    {"package", JSTokenStream::RESERVED},
    {"private", JSTokenStream::RESERVED},
    {"protected", JSTokenStream::RESERVED},
    {"public", JSTokenStream::RESERVED},
    {"short", JSTokenStream::RESERVED},
    {"static", JSTokenStream::RESERVED},
    {"super", JSTokenStream::RESERVED},
    {"synchronized", JSTokenStream::RESERVED},
    {"throw", JSTokenStream::THROW},
    {"throws", JSTokenStream::RESERVED},
    {"transient", JSTokenStream::RESERVED},
    {"try", JSTokenStream::TRY},
    {"volatile", JSTokenStream::RESERVED},
};
#ifdef VX
static JSKeywordHashtable *keywordHashtable;
#endif

/* Class data */
jsString_t *JSTokenStream::names;
int JSTokenStream::namesLength;

/* Procedures */
void JSTokenStream::checkNames()
{
#ifdef STATS
    if (names == NULL) {
#else
    if ((JSContext::printTrees || JSInterpreter::printICode)
            && names == NULL) {
#endif
        static jsString_t a[] = {
            "error",
            "eof",
            "eol",
            "popv",
            "enterwith",
            "leavewith",
            "return",
            "goto",
            "ifeq",
            "ifne",
            "dup",
            "setname",
            "bitor",
            "bitxor",
            "bitand",
            "eq",
            "ne",
            "lt",
            "le",
            "gt",
            "ge",
            "lsh",
            "rsh",
            "ursh",
            "add",
            "sub",
            "mul",
            "div",
            "mod",
            "bitnot",
            "neg",
            "new",
            "delprop",
            "typeof",
            "nameinc",
            "propinc",
            "eleminc",
            "namedec",
            "propdec",
            "elemdec",
            "getprop",
            "setprop",
            "getelem",
            "setelem",
            "call",
            "name",
            "number",
            "string",
            "zero",
            "one",
            "null",
            "this",
            "false",
            "true",
            "sheq",
            "shne",
            "closure",
            "object",
            "pop",
            "pos",
            "varinc",
            "vardec",
            "bindname",
            "throw",
            "in",
            "instanceof",
            "gosub",
            "retsub",
            "callspecial",
            "getthis",
            "newtemp",
            "usetemp",
            "getbase",
            "getvar",
            "setvar",
            "undefined",
            "try",
            "endtry",
            "newscope",
            "typeofname",
            "enuminit",
            "enumnext",
            "getproto",
            "getparent",
            "setproto",
            "setparent",
            "scope",
            "getscopeparent",
            "jthrow",
            "semi",
            "lb",
            "rb",
            "lc",
            "rc",
            "lp",
            "rp",
            "comma",
            "assign",
            "hook",
            "colon",
            "or",
            "and",
            "eqop",
            "relop",
            "shop",
            "unaryop",
            "inc",
            "dec",
            "dot",
            "primary",
            "function",
            "export",
            "import",
            "if",
            "else",
            "switch",
            "case",
            "default",
            "while",
            "do",
            "for",
            "break",
            "continue",
            "var",
            "with",
            "catch",
            "finally",
            "reserved",
            "nop",
            "not",
            "pre",
            "post",
            "void",
            "block",
            "arraylit",
            "objlit",
            "label",
            "target",
            "loop",
            "enumdone",
            "exprstmt",
            "parent",
            "convert",
            "jsr",
            "newlocal",
            "uselocal",
            "script",
            "line",
            "sourcefile",
        };
        names = a;
        namesLength = TableSize(names);
    }
}
jsBool_t JSTokenStream::isAlpha(int c)
{
#ifdef FASTER
    return (isalpha(c) != 0);
#else
    return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'));
#endif
}

jsBool_t JSTokenStream::isDigit(int c)
{
#ifdef FASTER
    return (isdigit(c) != 0);
#else
    return (c >= '0' && c <= '9');
#endif
}

jsBool_t JSTokenStream::isXDigit(int c)
{
#ifdef FASTER
    return (isxdigit(c) != 0);
#else
    return ((c >= '0' && c <= '9')
            || (c >= 'a' && c <= 'f')
            || (c >= 'A' && c <= 'F'));
#endif
}

jsBool_t JSTokenStream::isJSIdentifier(jsString_t s)
{
    int length = strlen(s);

    if (length == 0 || !isJavaIdentifierStart(s[0])) {
        return FALSE;
    }

    for (int i = 1; i < length; i++) {
        jsChar_t c = s[i];

        if (!isJavaIdentifierPart(c)) {
            if (c == '\\') {
                if (!((i + 5) < length)
                        && (s[i + 1] == 'u')
                        && isXDigit(s[i + 2])
                        && isXDigit(s[i + 3])
                        && isXDigit(s[i + 4])
                        && isXDigit(s[i + 5])) {
                    return FALSE;
                }
            }
        }
    }

    return TRUE;
}
int JSTokenStream::digit(jsChar_t c, int radix)
{
    if (c >= '0' && c <= '9') {
        return c - '0';
    }
    if (radix == 16) {
        c = tolower(c);
        if (c >= 'a' && c <= 'f') {
            return c - 'a' + 10;
        }
    }
    return 0;
}

jsBool_t JSTokenStream::isJavaIdentifierPart(int c)
{
    return isJavaIdentifierStart(c) || isDigit(c);
}

jsBool_t JSTokenStream::isJavaIdentifierStart(int c)
{
    return isAlpha(c) || (c == '$') || (c == '_');
}

void JSTokenStream::setString(JSContext * cx, jsString_t string)
{
#ifndef FASTER
    if (this->string != NULL)
        JS_DeleteZoneObj(cx, this->string);
#endif
    this->string = string;
}
jsString_t JSTokenStream::tokenToString(JSContext * cx, int token)
{
    if (JSContext::printTrees) {
        jsString_t string;
        int length;

        checkNames();
        if (token + 1 >= namesLength) {
            return NULL;
        }

        if (token == UNARYOP || token == ASSIGN || token == PRIMARY || token == EQOP || token == SHOP || token == RELOP) {
            length = strlen(names[token + 1]) + TableSize(" ") - 1 + strlen(names[this->op + 1]) + 1;
            string = (jsString_t) JS_NewZoneObj(cx, sizeof(*string) * length);

            sprintf(string, "%s %s", names[token + 1], names[this->op + 1]);
            return string;
        }

        if (token == STRING || token == OBJECT || token == NAME) {
            length = strlen(names[token + 1])
                    + strlen(this->string) + TableSize(" ` '") - 1 + 1;
            string = (jsString_t) JS_NewZoneObj(cx, sizeof(*string) * length);
            sprintf(string, "%s `%s '", names[token + 1], this->string);
            return string;
        }

        if (token == NUMBER) {
            length = TableSize("NUMBER ") - 1 + 32;
            string = (jsString_t) JS_NewZoneObj(cx, sizeof(*string) * length);
            sprintf(string, "NUMBER %f", this->number);
            return string;
        }

        return JS_NewZoneString(cx, names[token + 1]);
    }
    return JS_NewZoneString(cx, "");
}

jsString_t JSTokenStream::tokenToName(int type)
{
    checkNames();
    return names == NULL ? (jsString_t) "" : names[type + 1];
}

jsBool_t JSTokenStream::isJSSpace(int c)
{
    return (isspace(c) != 0);
}

jsBool_t JSTokenStream::isJSLineTerminator(int c)
{
    return (c == '\n' || c == '\r');
}

int JSTokenStream::stringToKeyword(jsString_t name)
{
#ifdef VX
    int value = keywordHashtable->get(name);

    if (value == 0) {
        return _EOF;
    }
    this->op = value >> 8;
    return value & 0xff;
#else /* !VX */
    class bsearchFuncs {
      public:
        static int compare(const void *a, const void *b) {
            jsString_t string = (jsString_t) a;
            JSKeyword *keyword = (JSKeyword *) b;

            return strcmp(string, keyword->string);
    }};

    JSKeyword *result;

    result = (JSKeyword *)
            bsearch(name, keywords, TableSize(keywords), sizeof(keywords[0]), &bsearchFuncs::compare);
    if (result == NULL) {
        return _EOF;
    }
    int x = result->value;

    this->op = x >> 8;
    return x & 0xff;
#endif /* VX */
}

JSScriptable *JSTokenStream::getScope()
{
    return scope;
}

/* return and pop the token from the stream if it matches...
 * otherwise return null
 */
jsBool_t JSTokenStream::matchToken(JSContext * cx, int toMatch)
{
    int token = getToken(cx);

    if (token == toMatch) {
        return TRUE;
    }
    // didn't match, push back token
    tokenno--;
    this->pushbackToken = token;
    return FALSE;
}

void JSTokenStream::clearPushback()
{
    this->pushbackToken = _EOF;
}
void JSTokenStream::ungetToken(JSContext * cx, int tt)
{
    if (this->pushbackToken != _EOF && tt != _ERROR) {
        jsString_t message = JSContext::getMessage(cx,
                "msg.token.replaces.pushback");
        cx->exceptionHandler->throwEx(new(cx) JSRuntimeException(message));
    }
    this->pushbackToken = tt;
    tokenno--;
}

int JSTokenStream::peekToken(JSContext * cx)
{
    int result = getToken(cx);

    this->pushbackToken = result;
    tokenno--;
    return result;
}

int JSTokenStream::peekTokenSameLine(JSContext * cx)
{
    int result;

    flags |= TSF_NEWLINES;  // SCAN_NEWLINES from jsscan.h
    result = peekToken(cx);
    flags &= ~TSF_NEWLINES; // HIDE_NEWLINES from jsscan.h
    if (this->pushbackToken == EOL) {
        this->pushbackToken = _EOF;
    }
    return result;
}

/* As defined in ECMA.  jsscan.c uses C isspace() (which allows
 * \v, I think.)  note that code in in->read() implicitly accepts
 * '\r' == \u000D as well.
 */
int JSTokenStream::getToken(JSContext * cx)
{
    int c;

    tokenno++;

    // Check for pushed-back token
    if (this->pushbackToken != _EOF) {
        int result = this->pushbackToken;

        this->pushbackToken = _EOF;
        return result;
    }

    // Eat whitespace, possibly sensitive to newlines.
    do {
        c = in.read(cx);
        if (c == '\n') {
            if ((flags & TSF_NEWLINES) != 0) {
                break;
            }
        }
    } while (isJSSpace(c) || c == '\n');

    if (c == EOF_CHAR) {
        return _EOF;
    }

    // identifier/keyword/instanceof?
    // watch out for starting with a <backslash>
    jsBool_t isUnicodeEscapeStart = FALSE;

    if (c == '\\') {
        c = in.read(cx);
        if (c == 'u') {
            isUnicodeEscapeStart = TRUE;
        }
        else {
            c = '\\';
        }
        // always unread the 'u' or whatever, we need 
        // to start the string below at the <backslash>.
        in.unread();
    }
    if (isUnicodeEscapeStart || isJavaIdentifierStart((jsChar_t) c)) {
        in.startString();

        jsBool_t containsEscape = isUnicodeEscapeStart;
        do {
            c = in.read(cx);
            if (c == '\\') {
                c = in.read(cx);
                containsEscape = (c == 'u');
            }
        } while (isJavaIdentifierPart((jsChar_t) c));
        in.unread();

        int result;

        jsString_t str = in.getString(cx);
        if (!str) {
            reportSyntaxError(cx, "msg.invalid.cx");
            return _ERROR;
        }
        // OPT we shouldn't have to make a string (object!) to
        // check if it's a keyword.

        // strictly speaking we should probably push-back
        // all the bad characters if the <backslash>uXXXX  
        // sequence is malformed. But since there isn't a  
        // correct context(is there?) for a bad Unicode  
        // escape sequence after an identifier, we can report
        // an error here.
        if (containsEscape) {
            jsString_t ca = str;
            JSStringBuffer x(cx);
            jsString_t s = strstr(str, "\\u");
            int escStart = s != NULL ? (s - str) : -1;
            int start = 0;
            int length = strlen(str);

            while (escStart != -1) {
                x.append(ca, start, escStart);

                jsBool_t goodEscape = FALSE;

                if ((escStart + 5) < length) {
                    if (isXDigit(ca[escStart + 2])) {
                        int val = digit(ca[escStart + 2], 16);
                        if (isXDigit(ca[escStart + 3])) {
                            val = (val << 4) | digit(ca[escStart + 3], 16);
                            if (isXDigit(ca[escStart + 4])) {
                                val = (val << 4) | digit(ca[escStart + 4], 16);
                                if (isXDigit(ca[escStart + 5])) {
                                    val = (val << 4)
                                            | digit(ca[escStart + 5], 16);
                                    x.append((jsChar_t) val);
                                    start = escStart + 6;
                                    goodEscape = TRUE;
                                }
                            }
                        }
                    }
                }
                if (!goodEscape) {
                    reportSyntaxError(cx, "msg.invalid.escape");
                    return _ERROR;
                }
                escStart = (s = strstr(str + start, "\\u")) != NULL ? (s - str) : -1;
            }
            x.append(ca, start, length - start);
#ifndef FASTER
            JS_DeleteZoneObj(cx, str);
#endif
            str = x.offString();
        }
        else {
            // Return the corresponding token if it's a keyword
            if ((result = stringToKeyword(str)) != _EOF) {
#ifndef FASTER
                JS_DeleteZoneObj(cx, str);
#endif
                return result;
            }
        }

        setString(cx, str);
        return NAME;
    }

    // is it a number?
    if (isDigit(c) || (c == '.' && isDigit(in.peek(cx)))) {
        int base = 10;
        in.startString();

        double dval = JSScriptRuntime::NaN;
        int64_t longval = 0;
        jsBool_t isInteger = TRUE;

        if (c == '0') {
            c = in.read(cx);
            if (c == 'x' || c == 'X') {
                c = in.read(cx);
                base = 16;
                // restart the string, losing leading 0x
                in.startString();
            }
            else if (isDigit(c)) {
                if (c < '8') {
                    base = 8;
                    // Restart the string, losing the leading 0
                    in.startString();
                }
                else {
                    /* Checking against c < '8' is non-ECMA, but
                     * is required to support legacy code; we've
                     * supported it in the past, and it's likely
                     * that "08" and "09" are in use in code
                     * having to do with dates.  So we need to
                     * support it, which makes our behavior a
                     * superset of ECMA in this area.  We raise a
                     * warning if a non-octal digit is
                     * encountered, then proceed as if it were
                     * decimal.
                     */
                    JSContext::reportWarning(cx, JSContext::getMessage(cx, "msg.bad.octal.literal %c", c), getSourceName(), in.getLineno(), getLine(cx), getOffset());
                    // implicitly retain the leading 0
                }
            }
            else {
                // implicitly retain the leading 0
            }
        }

        while (isXDigit(c)) {
            if (base < 16 && (isAlpha(c) || (base == 8 && c >= '8'))) {
                break;
            }
            c = in.read(cx);
        }

        if (base == 10 && (c == '.' || c == 'e' || c == 'E')) {
            isInteger = FALSE;
            if (c == '.') {
                do {
                    c = in.read(cx);
                } while (isDigit(c));
            }

            if (c == 'e' || c == 'E') {
                c = in.read(cx);
                if (c == '+' || c == '-') {
                    c = in.read(cx);
                }
                if (!isDigit(c)) {
                    in.getString(cx);   // throw away string in progress
                    reportSyntaxError(cx, "msg.missing.exponent");
                    return _ERROR;
                }
                do {
                    c = in.read(cx);
                } while (isDigit(c));
            }
        }
        in.unread();
        jsString_t numString = in.getString(cx);

        if (!numString) {
            return _ERROR;
        }

        if (base == 10 && !isInteger) {
            if (sscanf(numString, "%lf", &dval) != 1) {
                reportSyntaxError(cx, "msg.caught.nfe %s", numString);
                return _ERROR;
            }
        }
        else {
            dval = JSScriptRuntime::stringToNumber(numString, 0, base);
            longval = (int64_t) dval;

            // is it an integral fits-in-a-long value?
            if (longval != dval) {
                isInteger = FALSE;
            }
        }

        if (!isInteger) {
            /* Can't handle floats right now, because postfix INC/DEC
               generate Doubles, but I would generate a Float through this
               path, and it causes a stack mismatch. FIXME (MS)
               if (Float.MIN_VALUE <= dval && dval <= Float.MAX_VALUE)
               this->number = new Float((float) dval);
               else
             */
            this->number = (double)dval;
        }
        else {
            // We generate the smallest possible type here
            if (SCHAR_MIN <= longval && longval <= SCHAR_MAX) {
                this->number = (int8_t) longval;
            }
            else if (SHRT_MIN <= longval && longval <= SHRT_MAX) {
                this->number = (short)longval;
            }
            else if (INT_MIN <= longval && longval <= INT_MAX) {
                this->number = (int)longval;
            }
            else {
                // May lose some precision here, but that's the 
                // appropriate semantics.
                this->number = (double)longval;
            }
        }
#ifndef FASTER
        JS_DeleteZoneObj(cx, numString);
#endif
        return NUMBER;
    }

    // is it a string?
    if (c == '"' || c == '\'') {
        // We attempt to accumulate a string the fast way, by
        // building it directly out of the reader.  But if there
        // are any escaped characters in the string, we revert to
        // building it out of a StringBuffer.

        JSStringBuffer stringBuf(cx, 0);
        jsBool_t used = FALSE;

        int quoteChar = c;
        int val = 0;

        c = in.read(cx);
        in.startString();   // start after the first "
        while (c != quoteChar) {
            if (c == '\n' || c == EOF_CHAR) {
                in.unread();
                in.getString(cx);   // throw away the string in progress
                reportSyntaxError(cx, "msg.unterminated.string.lit", NULL);
                return _ERROR;
            }

            if (c == '\\') {
                // We've hit an escaped character; revert to the
                // slow method of building a string.
                // Don't include the backslash
                if (!used) {
                    in.unread();

                    jsString_t s = in.getString(cx);

                    if (!s) {
                        return _ERROR;
                    }

                    stringBuf.append(s);
#ifndef FASTER
                    JS_DeleteZoneObj(cx, s);
#endif
                    in.read(cx);
                    used = TRUE;
                }

                switch (c = in.read(cx)) {
                case 'b':
                    c = '\b';
                    break;
                case 'f':
                    c = '\f';
                    break;
                case 'n':
                    c = '\n';
                    break;
                case 'r':
                    c = '\r';
                    break;
                case 't':
                    c = '\t';
                    break;
                case 'v':
                    c = 0x0b;
                    break;
                    // case 'v': c = '\u000B'; break;
                    // \v a late addition to the ECMA spec.
                    // '\v' doesn't seem to be valid Java.

                default:
                    if (isDigit(c) && c < '8') {
                        val = c - '0';
                        c = in.read(cx);
                        if (isDigit(c) && c < '8') {
                            val = 8 * val + c - '0';
                            c = in.read(cx);
                            if (isDigit(c) && c < '8') {
                                val = 8 * val + c - '0';
                                c = in.read(cx);
                            }
                        }
                        in.unread();
                        if (val > 0377) {
                            reportSyntaxError(cx, "msg.oct.esc.too.large");
                            return _ERROR;
                        }
                        c = val;
                    }
                    else if (c == 'u') {
                        /*
                         * Get 4 hex digits; if the u escape is not
                         * followed by 4 hex digits, use 'u' + the literal
                         * character sequence that follows.  Do some manual
                         * match (OK because we're in a string) to avoid
                         * multi-char match on the underlying stream.
                         */
                        int c1, c2, c3, c4;

                        c1 = in.read(cx);
                        if (!isXDigit(c1)) {
                            in.unread();
                            c = 'u';
                        }
                        else {
                            val = digit((jsChar_t) c1, 16);
                            c2 = in.read(cx);
                            if (!isXDigit(c2)) {
                                in.unread();
                                stringBuf.append('u');
                                c = c1;
                            }
                            else {
                                val = 16 * val + digit((jsChar_t) c2, 16);
                                c3 = in.read(cx);
                                if (!isXDigit(c3)) {
                                    in.unread();
                                    stringBuf.append('u');
                                    stringBuf.append((jsChar_t) c1);
                                    c = c2;
                                }
                                else {
                                    val = 16 * val + digit((jsChar_t) c3, 16);
                                    c4 = in.read(cx);
                                    if (!isXDigit(c4)) {
                                        in.unread();
                                        stringBuf.append('u');
                                        stringBuf.append((jsChar_t) c1);
                                        stringBuf.append((jsChar_t) c2);
                                        c = c3;
                                    }
                                    else {
                                        // got 4 hex digits! Woo Hoo!
                                        val = 16 * val + digit((jsChar_t) c4, 16);
                                        c = val;
                                    }
                                }
                            }
                        }
                    }
                    else if (c == 'x') {
                        /* Get 2 hex digits, defaulting to 'x' + literal
                         * sequence, as above.
                         */
                        int c1, c2;

                        c1 = in.read(cx);
                        if (!isXDigit(c1)) {
                            in.unread();
                            c = 'x';
                        }
                        else {
                            val = digit((jsChar_t) c1, 16);
                            c2 = in.read(cx);
                            if (!isXDigit(c2)) {
                                in.unread();
                                stringBuf.append('x');
                                c = c1;
                            }
                            else {
                                // got 2 hex digits
                                val = 16 * val + digit((jsChar_t) c2, 16);
                                c = val;
                            }
                        }
                    }
                }
            }

            if (used) {
#if defined(USE_UTF8) || defined(USE_SJIS)
                if (c < 0x80) {
                    stringBuf.append((jsChar_t) c);
                }
                else {
                    jsChar_t mb[JSStringRuntime::_MB_CUR_MAX];
                    int n;

                    n = JSStringRuntime::wctomb(mb, c);
                    stringBuf.append(mb, 0, n);
                }
#else /* !(defined(USE_UTF8) || defined(USE_SJIS)) */
                stringBuf.append((jsChar_t) c);
#endif /* defined(USE_UTF8) || defined(USE_SJIS) */
            }
            c = in.read(cx);
        }

        if (used) {
#ifdef FASTER
            if (this->_string != NULL) {
                JS_DeleteZoneObj(cx, this->_string);
            }
            this->_string = stringBuf.offString();
            setString(cx, this->_string);
#else
            setString(cx, stringBuf.offString());
#endif
        }
        else {
            in.unread();    // miss the trailing "
            setString(cx, in.getString(cx));
            in.read(cx);
        }
        return STRING;
    }

    switch (c) {
    case '\n':
        return EOL;
    case ';':
        return SEMI;
    case '[':
        return LB;
    case ']':
        return RB;
    case '{':
        return LC;
    case '}':
        return RC;
    case '(':
        return LP;
    case ')':
        return RP;
    case ',':
        return COMMA;
    case '?':
        return HOOK;
    case ':':
        return COLON;
    case '.':
        return DOT;

    case '|':
        if (in.match(cx, '|')) {
            return OR;
        }
        else if (in.match(cx, '=')) {
            this->op = BITOR;
            return ASSIGN;
        }
        else {
            return BITOR;
        }

    case '^':
        if (in.match(cx, '=')) {
            this->op = BITXOR;
            return ASSIGN;
        }
        else {
            return BITXOR;
        }

    case '&':
        if (in.match(cx, '&')) {
            return AND;
        }
        else if (in.match(cx, '=')) {
            this->op = BITAND;
            return ASSIGN;
        }
        else {
            return BITAND;
        }

    case '=':
        if (in.match(cx, '=')) {
            if (in.match(cx, '=')) {
                this->op = SHEQ;
            }
            else {
                this->op = EQ;
            }
            return EQOP;
        }
        else {
            this->op = NOP;
            return ASSIGN;
        }

    case '!':
        if (in.match(cx, '=')) {
            if (in.match(cx, '=')) {
                this->op = SHNE;
            }
            else {
                this->op = NE;
            }
            return EQOP;
        }
        else {
            this->op = NOT;
            return UNARYOP;
        }

    case '<':
        /* NB:treat HTML begin-comment as comment-till-eol */
        if (in.match(cx, '!')) {
            if (in.match(cx, '-')) {
                if (in.match(cx, '-')) {
                    while ((c = in.read(cx)) != EOF_CHAR && c != '\n')
                        /* skip to end of line */ ;
                    in.unread();
                    return getToken(cx);    // in place of 'goto retry'
                }
                in.unread();
            }
            in.unread();
        }
        if (in.match(cx, '<')) {
            if (in.match(cx, '=')) {
                this->op = LSH;
                return ASSIGN;
            }
            else {
                this->op = LSH;
                return SHOP;
            }
        }
        else {
            if (in.match(cx, '=')) {
                this->op = LE;
                return RELOP;
            }
            else {
                this->op = LT;
                return RELOP;
            }
        }

    case '>':
        if (in.match(cx, '>')) {
            if (in.match(cx, '>')) {
                if (in.match(cx, '=')) {
                    this->op = URSH;
                    return ASSIGN;
                }
                else {
                    this->op = URSH;
                    return SHOP;
                }
            }
            else {
                if (in.match(cx, '=')) {
                    this->op = RSH;
                    return ASSIGN;
                }
                else {
                    this->op = RSH;
                    return SHOP;
                }
            }
        }
        else {
            if (in.match(cx, '=')) {
                this->op = GE;
                return RELOP;
            }
            else {
                this->op = GT;
                return RELOP;
            }
        }

    case '*':
        if (in.match(cx, '=')) {
            this->op = MUL;
            return ASSIGN;
        }
        else {
            return MUL;
        }

    case '/':
        // is it a // comment?
        if (in.match(cx, '/')) {
            while ((c = in.read(cx)) != EOF_CHAR && c != '\n')
                /* skip to end of line */ ;
            in.unread();
            return getToken(cx);
        }
        if (in.match(cx, '*')) {
            while ((c = in.read(cx)) != -1 && !(c == '*' && in.match(cx, '/'))) {
                if (c == '\n') {
                }
                else if (c == '/' && in.match(cx, '*')) {
                    if (in.match(cx, '/')) {
                        return getToken(cx);
                    }
                    reportSyntaxError(cx, "msg.nested.comment", NULL);
                    return _ERROR;
                }
            }
            if (c == EOF_CHAR) {
                reportSyntaxError(cx, "msg.unterminated.comment", NULL);
                return _ERROR;
            }
            return getToken(cx);    // `goto retry'
        }

        // is it a regexp?
        if ((flags & TSF_REGEXP) != 0) {
            // We don't try to use the in.startString/in.getString
            // approach, because escaped characters (which break it)
            // seem likely to be common.
            JSStringBuffer re(cx);

            while ((c = in.read(cx)) != '/') {
                if (c == '\n' || c == EOF_CHAR) {
                    in.unread();
                    reportSyntaxError(cx, "msg.unterminated.re.lit", NULL);
                    return _ERROR;
                }
                if (c == '\\') {
                    re.append((jsChar_t) c);
                    c = in.read(cx);
                }

                re.append((jsChar_t) c);
            }

            JSStringBuffer flagsBuf(cx);

            while (TRUE) {
                if (in.match(cx, 'g')) {
                    flagsBuf.append('g');
                }
                else if (in.match(cx, 'i')) {
                    flagsBuf.append('i');
                }
                else if (in.match(cx, 'm')) {
                    flagsBuf.append('m');
                }
                else {
                    break;
                }
            }

            if (isAlpha(in.peek(cx))) {
                reportSyntaxError(cx, "msg.invalid.re.flag", NULL);
                return _ERROR;
            }

#ifdef FASTER
            if (this->regExp != NULL) {
                JS_DeleteZoneObj(cx, this->regExp);
            }
            this->regExp = re.offString();
            setString(cx, this->regExp);
#else
            setString(cx, re.offString());
#endif
            if (this->regExpFlags != NULL) {
                JS_DeleteZoneObj(cx, this->regExpFlags);
            }
            this->regExpFlags = flagsBuf.offString();
            return OBJECT;
        }


        if (in.match(cx, '=')) {
            this->op = DIV;
            return ASSIGN;
        }
        else {
            return DIV;
        }

    case '%':
        this->op = MOD;
        if (in.match(cx, '=')) {
            return ASSIGN;
        }
        else {
            return MOD;
        }

    case '~':
        this->op = BITNOT;
        return UNARYOP;

    case '+':
    case '-':
        if (in.match(cx, '=')) {
            if (c == '+') {
                this->op = ADD;
                return ASSIGN;
            }
            else {
                this->op = SUB;
                return ASSIGN;
            }
        }
        else if (in.match(cx, (jsChar_t) c)) {
            if (c == '+') {
                return INC;
            }
            else {
                return DEC;
            }
        }
        else if (c == '-') {
            return SUB;
        }
        else {
            return ADD;
        }

    default:
        reportSyntaxError(cx, "msg.illegal.character", NULL);
        return _ERROR;
    }
}
void JSTokenStream::reportSyntaxError(JSContext * cx, jsString_t messageProperty, ...)
{
    va_list ap;

    va_start(ap, messageProperty);

    jsString_t message = JSContext::getMessage(cx, messageProperty, ap);

    va_end(ap);

    if (scope != NULL) {
        // We're probably in an eval. Need to throw an exception.
        cx->exceptionHandler->throwEx(JSNativeGlobal::constructError(cx, "SyntaxError", message, scope, getSourceName(), getLineno(), getOffset(), getLine(cx)));
    }
    else {
        JSContext::reportError(cx, message, getSourceName(), getLineno(), getLine(cx), getOffset());
    }
}
void JSTokenStream::destroy(JSContext * cx)
{
#ifdef FASTER
    if (_string != NULL)
        JS_DeleteZoneObj(cx, _string);
    if (regExp != NULL)
        JS_DeleteZoneObj(cx, regExp);
#else
    if (string != NULL)
        JS_DeleteZoneObj(cx, string);
#endif
    in.destroy(cx);
    JSZoneNew::destroy(cx);
}

JSTokenStream::JSTokenStream(JSContext * cx, JSReader * in, JSScriptable * scope, jsString_t sourceName, int lineno)
:in(cx, in, lineno)
{
    this->scope = scope;
    this->pushbackToken = _EOF;
    this->sourceName = sourceName;
    flags = 0;
    string = NULL;
    tokenno = 1;    //avoid purify UMR error
#ifdef FASTER
    _string = NULL;
    regExp = NULL;
#endif
    regExpFlags = NULL;
}

void JSTokenStream::classInitialize()
{
#ifdef VX
    static JSKeywordAllocator allocator;

    keywordHashtable = new JSKeywordHashtable(&allocator);
    if (keywordHashtable == NULL) {
        return;
    }
    for (int i = 0; i < (int)TableSize(keywords); ++i) {
        keywordHashtable->put(keywords[i].string, keywords[i].value);
    }
#else /* !VX */
    class qsortFuncs {
      public:
        static int compare(const void *a, const void *b) {
            JSKeyword *kwa = (JSKeyword *) a;
            JSKeyword *kwb = (JSKeyword *) b;

            return strcmp(kwa->string, kwb->string);
    }};

    qsort(keywords, TableSize(keywords), sizeof(keywords[0]), &qsortFuncs::compare);
#endif /* VX */
}
