alvinalexander.com | career | drupal | java | mac | mysql | perl | scala | uml | unix  

HSQLDB example source code file (Tokenizer.java)

This example HSQLDB source code file (Tokenizer.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" TM.

Java - HSQLDB tags/keywords

decimal, hsqlexception, hsqlexception, long_name, long_name, math, no_type, number, querying, quoted_identifier, special, special, string, string, util, wait

The HSQLDB Tokenizer.java source code

/*
 * For work developed by the HSQL Development Group:
 *
 * Copyright (c) 2001-2010, The HSQL Development Group
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * Neither the name of the HSQL Development Group nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *
 *
 * For work originally developed by the Hypersonic SQL Group:
 *
 * Copyright (c) 1995-2000, The Hypersonic SQL Group.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * Neither the name of the Hypersonic SQL Group nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP,
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * This software consists of voluntary contributions made by many individuals
 * on behalf of the Hypersonic SQL Group.
 */


package org.hsqldb;

import java.math.BigDecimal;
import java.util.Locale;

import org.hsqldb.lib.IntValueHashMap;
import org.hsqldb.store.ValuePool;
import org.hsqldb.lib.java.JavaSystem;

// fredt@users 20020218 - patch 455785 by hjbusch@users - large DECIMAL inserts
// also Long.MIM_VALUE (bug 473388) inserts - applied to different parts
// fredt@users 20020408 - patch 1.7.0 by fredt - exact integral types
// integral values are cast into the smallest type that can hold them
// fredt@users 20020501 - patch 550970 by boucherb@users - fewer StringBuffers
// fredt@users 20020611 - patch 1.7.0 by fredt - correct statement logging
// changes to the working of getLastPart() to return the correct statement for
// logging in the .script file.
// also restructuring to reduce use of objects and speed up tokenising of
// strings and quoted identifiers
// fredt@users 20021112 - patch 1.7.2 by Nitin Chauhan - use of switch
// rewrite of the majority of multiple if(){}else{} chains with switch(){}
// fredt@users 20030610 - patch 1.7.2 - no StringBuffers

/**
 * Provides the ability to tokenize SQL character sequences.
 *
 * Extensively rewritten and extended in successive versions of HSQLDB.
 *
 * @author Thomas Mueller (Hypersonic SQL Group)
 * @version 1.8.0
 * @since Hypersonic SQL
 */
public class Tokenizer {

    private static final int NO_TYPE     = 0,
                             NAME        = 1,
                             LONG_NAME   = 2,
                             SPECIAL     = 3,
                             NUMBER      = 4,
                             FLOAT       = 5,
                             STRING      = 6,
                             LONG        = 7,
                             DECIMAL     = 8,
                             BOOLEAN     = 9,
                             DATE        = 10,
                             TIME        = 11,
                             TIMESTAMP   = 12,
                             NULL        = 13,
                             NAMED_PARAM = 14;

    // used only internally
    private static final int QUOTED_IDENTIFIER = 15,
                             REMARK_LINE       = 16,
                             REMARK            = 17;
    private String           sCommand;
    private int              iLength;
    private int              iIndex;
    private int              tokenIndex;
    private int              nextTokenIndex;
    private int              beginIndex;
    private int              iType;
    private String           sToken;
    private int              indexLongNameFirst = -1;
    private String           sLongNameFirst = null;
    private int              typeLongNameFirst;

    // getToken() will clear LongNameFirst unless retainFirst is set.
    private boolean retainFirst = false;

//    private String           sLongNameLast;
    // WAIT.  Don't do anything before popping another Token (because the
    // state variables aren't set properly due to a call of wait()).
    private boolean bWait;
    private boolean lastTokenQuotedID;

    // literals that are values
    static IntValueHashMap valueTokens;

    static {
        valueTokens = new IntValueHashMap();

        valueTokens.put(Token.T_NULL, NULL);
        valueTokens.put(Token.T_TRUE, BOOLEAN);
        valueTokens.put(Token.T_FALSE, BOOLEAN);
    }

    public Tokenizer() {}

    public Tokenizer(String s) {

        sCommand = s;
        iLength  = s.length();
        iIndex   = 0;
    }

    public void reset(String s) {

        sCommand           = s;
        iLength            = s.length();
        iIndex             = 0;
        tokenIndex         = 0;
        nextTokenIndex     = 0;
        beginIndex         = 0;
        iType              = NO_TYPE;
        typeLongNameFirst  = NO_TYPE;
        sToken             = null;
        indexLongNameFirst = -1;
        sLongNameFirst     = null;

//        sLongNameLast  = null;
        bWait             = false;
        lastTokenQuotedID = false;
        retainFirst       = false;
    }

    /**
     *
     * @throws HsqlException
     */
    void back() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        nextTokenIndex = iIndex;
        iIndex         = ( indexLongNameFirst != -1 ) ? indexLongNameFirst : tokenIndex;
        bWait          = true;
    }

    /**
     * get the given token or throw
     *
     * for commands and simple unquoted identifiers only
     *
     * @param match
     *
     * @throws HsqlException
     */
    String getThis(String match) throws HsqlException {

        getToken();
        matchThis(match);

        return sToken;
    }

    /**
     * for commands and simple unquoted identifiers only
     */
    void matchThis(String match) throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        if (!sToken.equals(match) || iType == QUOTED_IDENTIFIER
                || iType == LONG_NAME) {
            String token = iType == LONG_NAME ? sLongNameFirst
                                              : sToken;

            throw Trace.error(Trace.UNEXPECTED_TOKEN, Trace.TOKEN_REQUIRED,
                              new Object[] {
                token, match
            });
        }
    }

    void throwUnexpected() throws HsqlException {

        String token = iType == LONG_NAME ? sLongNameFirst
                                          : sToken;

        throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
    }

    /**
     * Used for commands only
     *
     *
     * @param match
     */
    public boolean isGetThis(String match) throws HsqlException {

        getToken();

        if (iType != QUOTED_IDENTIFIER && iType != LONG_NAME
                && sToken.equals(match)) {
            return true;
        }

        back();

        return false;
    }

    /**
     * this methode is called before other wasXXX methods and takes
     * precedence
     */
    boolean wasValue() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        switch (iType) {

            case STRING :
            case NUMBER :
            case LONG :
            case FLOAT :
            case DECIMAL :
            case BOOLEAN :
            case NULL :
                return true;

            default :
                return false;
        }
    }

    boolean wasQuotedIdentifier() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return lastTokenQuotedID;

        // iType won't help for LONG_NAMEs.
        //return iType == QUOTED_IDENTIFIER;
    }

    boolean wasFirstQuotedIdentifier() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return (typeLongNameFirst == QUOTED_IDENTIFIER);
    }

    /**
     * Method declaration
     *
     *
     * @return
     */
    boolean wasLongName() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return iType == LONG_NAME;
    }

    /**
     * Simple Name means a quoted or unquoted identifier without
     * qualifiers provided it is not in the hKeyword list.
     *
     * @return
     */
    boolean wasSimpleName() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        if (iType == QUOTED_IDENTIFIER && sToken.length() != 0) {
            return true;
        }

        if (iType != NAME) {
            return false;
        }

        return !Token.isKeyword(sToken);
    }

    /**
     * checks whether the previously obtained token was a (named) parameter
     *
     * @return true if the previously obtained token was a (named) parameter
     */
    boolean wasParameter() throws HsqlException {

        Trace.doAssert(!bWait, "Querying state when in Wait mode");

        return (iType == NAMED_PARAM);
    }

    /**
     * Name means all quoted and unquoted identifiers plus any word not in the
     * hKeyword list.
     *
     * @return true if it's a name
     */
    boolean wasName() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        if (iType == QUOTED_IDENTIFIER) {
            return true;
        }

        if (iType != NAME && iType != LONG_NAME) {
            return false;
        }

        return !Token.isKeyword(sToken);
    }

    String getLongNamePre() throws HsqlException {
        return null;
    }

    /**
     * Return first part of long name
     *
     *
     * @return
     */
    String getLongNameFirst() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        return sLongNameFirst;
    }

    boolean wasSimpleToken() throws HsqlException {
        return iType != QUOTED_IDENTIFIER && iType != LONG_NAME
               && iType != STRING && iType != NAMED_PARAM;
    }

    String getSimpleToken() throws HsqlException {

        getToken();

        if (!wasSimpleToken()) {
            String token = iType == LONG_NAME ? sLongNameFirst
                                              : sToken;

            throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
        }

        return sToken;
    }

    public boolean wasThis(String match) throws HsqlException {

        if (sToken.equals(match) && iType != QUOTED_IDENTIFIER
                && iType != LONG_NAME && iType != STRING) {
            return true;
        }

        return false;
    }

    /**
     * getName() is more broad than getSimpleName() in that it includes
     * 2-part names as well
     *
     * @return popped name
     * @throws HsqlException if next token is not an AName
     */
    public String getName() throws HsqlException {

        getToken();

        if (!wasName()) {
            throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
        }

        return sToken;
    }

    /**
     * Returns a single, unqualified name (identifier)
     *
     * @return name
     * @throws HsqlException
     */
    public String getSimpleName() throws HsqlException {

        getToken();

        if (!wasSimpleName()) {
            String token = iType == LONG_NAME ? sLongNameFirst
                                              : sToken;

            throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
        }

        return sToken;
    }

    /**
     * Return any token.
     *
     *
     * @return
     *
     * @throws HsqlException
     */
    public String getString() throws HsqlException {

        getToken();

        return sToken;
    }

    int getInt() throws HsqlException {

        long v = getBigint();

        if (v > Integer.MAX_VALUE || v < Integer.MIN_VALUE) {
            throw Trace.error(Trace.WRONG_DATA_TYPE,
                              Types.getTypeString(getType()));
        }

        return (int) v;
    }

    static BigDecimal LONG_MAX_VALUE_INCREMENT =
        BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.valueOf(1));

    long getBigint() throws HsqlException {

        boolean minus = false;

        getToken();

        if (sToken.equals("-")) {
            minus = true;

            getToken();
        }

        Object o = getAsValue();
        int    t = getType();

        switch (t) {

            case Types.INTEGER :
            case Types.BIGINT :
                break;

            case Types.DECIMAL :

                // only Long.MAX_VALUE + 1 together with minus is acceptable
                if (minus && LONG_MAX_VALUE_INCREMENT.equals(o)) {
                    return Long.MIN_VALUE;
                }
            default :
                throw Trace.error(Trace.WRONG_DATA_TYPE,
                                  Types.getTypeString(t));
        }

        long v = ((Number) o).longValue();

        return minus ? -v
                     : v;
    }

    Object getInType(int type) throws HsqlException {

        getToken();

        Object o = getAsValue();
        int    t = getType();

        if (t != type) {
            throw Trace.error(Trace.WRONG_DATA_TYPE, Types.getTypeString(t));
        }

        return o;
    }

    /**
     *
     *
     *
     * @return
     */
    public int getType() throws HsqlException {

        if (bWait) {
            Trace.doAssert(false, "Querying state when in Wait mode");
        }

        // todo: make sure it's used only for Values!
        // todo: synchronize iType with hColumn
        switch (iType) {

            case STRING :
                return Types.VARCHAR;

            case NUMBER :
                return Types.INTEGER;

            case LONG :
                return Types.BIGINT;

            case FLOAT :
                return Types.DOUBLE;

            case DECIMAL :
                return Types.DECIMAL;

            case BOOLEAN :
                return Types.BOOLEAN;

            case DATE :
                return Types.DATE;

            case TIME :
                return Types.TIME;

            case TIMESTAMP :
                return Types.TIMESTAMP;

            default :
                return Types.NULL;
        }
    }

    /**
     * Method declaration
     *
     *
     * @return
     *
     * @throws HsqlException
     */
    Object getAsValue() throws HsqlException {

        if (!wasValue()) {
            throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
        }

        switch (iType) {

            case NULL :
                return null;

            case STRING :

                //fredt - no longer returning string with a singlequote as last char
                return sToken;

            case LONG :
                return ValuePool.getLong(Long.parseLong(sToken));

            case NUMBER :

                // fredt - this returns unsigned values which are later negated.
                // as a result Integer.MIN_VALUE or Long.MIN_VALUE are promoted
                // to a wider type.
                if (sToken.length() < 11) {
                    try {
                        return ValuePool.getInt(Integer.parseInt(sToken));
                    } catch (Exception e1) {}
                }

                if (sToken.length() < 20) {
                    try {
                        iType = LONG;

                        return ValuePool.getLong(Long.parseLong(sToken));
                    } catch (Exception e2) {}
                }

                iType = DECIMAL;

                return new BigDecimal(sToken);

            case FLOAT :
                double d = JavaSystem.parseDouble(sToken);
                long   l = Double.doubleToLongBits(d);

                return ValuePool.getDouble(l);

            case DECIMAL :
                return new BigDecimal(sToken);

            case BOOLEAN :
                return sToken.equalsIgnoreCase("TRUE") ? Boolean.TRUE
                                                       : Boolean.FALSE;

            case DATE :
                return HsqlDateTime.dateValue(sToken);

            case TIME :
                return HsqlDateTime.timeValue(sToken);

            case TIMESTAMP :
                return HsqlDateTime.timestampValue(sToken);

            default :
                return sToken;
        }
    }

    /**
     * return the current position to be used for VIEW processing
     *
     * @return
     */
    int getPosition() {
        return iIndex;
    }

    /**
     * mark the current position to be used for future getLastPart() calls
     *
     * @return
     */
    String getPart(int begin, int end) {
        return sCommand.substring(begin, end);
    }

    /**
     * mark the current position to be used for future getLastPart() calls
     *
     * @return
     */
    int getPartMarker() {
        return beginIndex;
    }

    /**
     * mark the current position to be used for future getLastPart() calls
     *
     */
    void setPartMarker() {
        beginIndex = iIndex;
    }

    /**
     * mark the position to be used for future getLastPart() calls
     *
     */
    void setPartMarker(int position) {
        beginIndex = position;
    }

    /**
     * return part of the command string from the last marked position
     *
     * @return
     */
    String getLastPart() {
        return sCommand.substring(beginIndex, iIndex);
    }

// fredt@users 20020910 - patch 1.7.1 by Nitin Chauhan - rewrite as switch

    /**
     * Method declaration
     *
     *
     * @throws HsqlException
     */
    private void getToken() throws HsqlException {

        if (bWait) {
            bWait  = false;
            iIndex = nextTokenIndex;

            return;
        }

        if (!retainFirst) {
            sLongNameFirst     = null;
            indexLongNameFirst = -1;
            typeLongNameFirst  = NO_TYPE;
        }

        while (iIndex < iLength
                && Character.isWhitespace(sCommand.charAt(iIndex))) {
            iIndex++;
        }

        sToken     = "";
        tokenIndex = iIndex;

        if (iIndex >= iLength) {
            iType = NO_TYPE;

            return;
        }

        char    c        = sCommand.charAt(iIndex);
        boolean point    = false,
                digit    = false,
                exp      = false,
                afterexp = false;
        boolean end      = false;
        char    cfirst   = 0;

        lastTokenQuotedID = false;

        if (Character.isJavaIdentifierStart(c)) {
            iType = NAME;
        } else if (Character.isDigit(c)) {
            iType = NUMBER;
            digit = true;
        } else {
            switch (c) {

                case '(' :
                    sToken = Token.T_OPENBRACKET;
                    iType  = SPECIAL;

                    iIndex++;

                    return;

                case ')' :
                    sToken = Token.T_CLOSEBRACKET;
                    iType  = SPECIAL;

                    iIndex++;

                    return;

                case ',' :
                    sToken = Token.T_COMMA;
                    iType  = SPECIAL;

                    iIndex++;

                    return;

                case '*' :
                    sToken = Token.T_MULTIPLY;
                    iType  = SPECIAL;

                    iIndex++;

                    return;

                case '=' :
                    sToken = Token.T_EQUALS;
                    iType  = SPECIAL;

                    iIndex++;

                    return;

                case ';' :
                    sToken = Token.T_SEMICOLON;
                    iType  = SPECIAL;

                    iIndex++;

                    return;

                case '+' :
                    sToken = Token.T_PLUS;
                    iType  = SPECIAL;

                    iIndex++;

                    return;

                case '%' :
                    sToken = Token.T_PERCENT;
                    iType  = SPECIAL;

                    iIndex++;

                    return;

                case '?' :
                    sToken = Token.T_QUESTION;
                    iType  = SPECIAL;

                    iIndex++;

                    return;

                case ':' :
                    Trace.check(++iIndex < iLength,
                                Trace.UNEXPECTED_END_OF_COMMAND);

                    c = sCommand.charAt(iIndex);

                    Trace.check(Character.isJavaIdentifierStart(c),
                                Trace.INVALID_IDENTIFIER, ":" + c);

                    iType = NAMED_PARAM;
                    break;

                case '\"' :
                    lastTokenQuotedID = true;
                    iType             = QUOTED_IDENTIFIER;

                    iIndex++;

                    sToken = getString('"');

                    if (iIndex == sCommand.length()) {
                        return;
                    }

                    c = sCommand.charAt(iIndex);

                    if (c == '.') {
                        sLongNameFirst     = sToken;
                        indexLongNameFirst = tokenIndex;
                        typeLongNameFirst  = iType;

                        iIndex++;

                        if (retainFirst) {
                            throw Trace.error(Trace.THREE_PART_IDENTIFIER);
                        }

// fredt - todo - avoid recursion - this has problems when there is whitespace
// after the dot - the same with NAME
                        retainFirst = true;

                        getToken();

                        retainFirst = false;
                        iType       = LONG_NAME;
                    }

                    return;

                case '\'' :
                    iType = STRING;

                    iIndex++;

                    sToken = getString('\'');

                    return;

                case '!' :
                case '<' :
                case '>' :
                case '|' :
                case '/' :
                case '-' :
                    cfirst = c;
                    iType  = SPECIAL;
                    break;

                case '.' :
                    iType = DECIMAL;
                    point = true;
                    break;

                default :
                    throw Trace.error(Trace.UNEXPECTED_TOKEN,
                                      String.valueOf(c));
            }
        }

        int start = iIndex++;

        while (true) {
            if (iIndex >= iLength) {
                c   = ' ';
                end = true;

                Trace.check(iType != STRING && iType != QUOTED_IDENTIFIER,
                            Trace.UNEXPECTED_END_OF_COMMAND);
            } else {
                c = sCommand.charAt(iIndex);
            }

            switch (iType) {

                case NAMED_PARAM :
                case NAME :
                    if (Character.isJavaIdentifierPart(c)) {
                        break;
                    }

                    // fredt - todo new char[] to back sToken
                    sToken =
                        sCommand.substring(start,
                                           iIndex).toUpperCase(Locale.ENGLISH);

                    // the following only for NAME, not for NAMED_PARAM
                    if (iType == NAMED_PARAM) {
                        return;
                    }

                    if (c == '.') {
                        typeLongNameFirst  = iType;
                        sLongNameFirst     = sToken;
                        indexLongNameFirst = tokenIndex;

                        iIndex++;

                        if (retainFirst) {
                            throw Trace.error(Trace.THREE_PART_IDENTIFIER);
                        }

                        retainFirst = true;

                        getToken();    // todo: eliminate recursion

                        retainFirst = false;
                        iType       = LONG_NAME;
                    } else if (c == '(') {

                        // it is a function call
                    } else {

                        // if in value list then it is a value
                        int type = valueTokens.get(sToken, -1);

                        if (type != -1) {
                            iType = type;
                        }
                    }

                    return;

                case QUOTED_IDENTIFIER :
                case STRING :

                    // shouldn't get here
                    break;

                case REMARK :
                    if (end) {

                        // unfinished remark
                        // maybe print error here
                        iType = NO_TYPE;

                        return;
                    } else if (c == '*') {
                        iIndex++;

                        if (iIndex < iLength
                                && sCommand.charAt(iIndex) == '/') {

                            // using recursion here
                            iIndex++;

                            getToken();

                            return;
                        }
                    }
                    break;

                case REMARK_LINE :
                    if (end) {
                        iType = NO_TYPE;

                        return;
                    } else if (c == '\r' || c == '\n') {

                        // using recursion here
                        getToken();

                        return;
                    }
                    break;

                case SPECIAL :
                    if (c == '/' && cfirst == '/') {
                        iType = REMARK_LINE;

                        break;
                    } else if (c == '-' && cfirst == '-') {
                        iType = REMARK_LINE;

                        break;
                    } else if (c == '*' && cfirst == '/') {
                        iType = REMARK;

                        break;
                    } else if (c == '>' || c == '=' || c == '|') {
                        break;
                    }

                    sToken = sCommand.substring(start, iIndex);

                    return;

                case NUMBER :
                case FLOAT :
                case DECIMAL :
                    if (Character.isDigit(c)) {
                        digit = true;
                    } else if (c == '.') {
                        iType = DECIMAL;

                        if (point) {
                            throw Trace.error(Trace.UNEXPECTED_TOKEN, ".");
                        }

                        point = true;
                    } else if (c == 'E' || c == 'e') {
                        if (exp) {
                            throw Trace.error(Trace.UNEXPECTED_TOKEN, "E");
                        }

                        // HJB-2001-08-2001 - now we are sure it's a float
                        iType = FLOAT;

                        // first character after exp may be + or -
                        afterexp = true;
                        point    = true;
                        exp      = true;
                    } else if (c == '-' && afterexp) {
                        afterexp = false;
                    } else if (c == '+' && afterexp) {
                        afterexp = false;
                    } else {
                        afterexp = false;

                        if (!digit) {
                            if (point && start == iIndex - 1) {
                                sToken = ".";
                                iType  = SPECIAL;

                                return;
                            }

                            throw Trace.error(Trace.UNEXPECTED_TOKEN,
                                              String.valueOf(c));
                        }

                        sToken = sCommand.substring(start, iIndex);

                        return;
                    }
            }

            iIndex++;
        }
    }

// fredt - strings are constructed from new char[] objects to avoid slack
// because these strings might end up as part of internal data structures
// or table elements.
// we may consider using pools to avoid recreating the strings
    private String getString(char quoteChar) throws HsqlException {

        try {
            int     nextIndex   = iIndex;
            boolean quoteInside = false;

            for (;;) {
                nextIndex = sCommand.indexOf(quoteChar, nextIndex);

                if (nextIndex < 0) {
                    throw Trace.error(Trace.UNEXPECTED_END_OF_COMMAND);
                }

                if (nextIndex < iLength - 1
                        && sCommand.charAt(nextIndex + 1) == quoteChar) {
                    quoteInside = true;
                    nextIndex   += 2;

                    continue;
                }

                break;
            }

            char[] chBuffer = new char[nextIndex - iIndex];

            sCommand.getChars(iIndex, nextIndex, chBuffer, 0);

            int j = chBuffer.length;

            if (quoteInside) {
                j = 0;

                // fredt - loop assumes all occurences of quoteChar are paired
                // this has already been checked by the preprocessing loop
                for (int i = 0; i < chBuffer.length; i++, j++) {
                    if (chBuffer[i] == quoteChar) {
                        i++;
                    }

                    chBuffer[j] = chBuffer[i];
                }
            }

            iIndex = ++nextIndex;

            return new String(chBuffer, 0, j);
        } catch (HsqlException e) {
            throw e;
        } catch (Exception e) {
            e.toString();
        }

        return null;
    }

    /**
     * Method declaration
     *
     *
     * @return
     */
    int getLength() {
        return iLength;
    }
}

Other HSQLDB examples (source code examples)

Here is a short list of links related to this HSQLDB Tokenizer.java source code file:

... this post is sponsored by my books ...

#1 New Release!

FP Best Seller

 

new blog posts

 

Copyright 1998-2021 Alvin Alexander, alvinalexander.com
All Rights Reserved.

A percentage of advertising revenue from
pages under the /java/jwarehouse URI on this website is
paid back to open source projects.