|
HSQLDB example source code file (Tokenizer.java)
The HSQLDB Tokenizer.java source code/* * For work developed by the HSQL Development Group: * * Copyright (c) 2001-2010, The HSQL Development Group * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * Neither the name of the HSQL Development Group nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * * * For work originally developed by the Hypersonic SQL Group: * * Copyright (c) 1995-2000, The Hypersonic SQL Group. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * Neither the name of the Hypersonic SQL Group nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * This software consists of voluntary contributions made by many individuals * on behalf of the Hypersonic SQL Group. */ package org.hsqldb; import java.math.BigDecimal; import java.util.Locale; import org.hsqldb.lib.IntValueHashMap; import org.hsqldb.store.ValuePool; import org.hsqldb.lib.java.JavaSystem; // fredt@users 20020218 - patch 455785 by hjbusch@users - large DECIMAL inserts // also Long.MIM_VALUE (bug 473388) inserts - applied to different parts // fredt@users 20020408 - patch 1.7.0 by fredt - exact integral types // integral values are cast into the smallest type that can hold them // fredt@users 20020501 - patch 550970 by boucherb@users - fewer StringBuffers // fredt@users 20020611 - patch 1.7.0 by fredt - correct statement logging // changes to the working of getLastPart() to return the correct statement for // logging in the .script file. // also restructuring to reduce use of objects and speed up tokenising of // strings and quoted identifiers // fredt@users 20021112 - patch 1.7.2 by Nitin Chauhan - use of switch // rewrite of the majority of multiple if(){}else{} chains with switch(){} // fredt@users 20030610 - patch 1.7.2 - no StringBuffers /** * Provides the ability to tokenize SQL character sequences. * * Extensively rewritten and extended in successive versions of HSQLDB. * * @author Thomas Mueller (Hypersonic SQL Group) * @version 1.8.0 * @since Hypersonic SQL */ public class Tokenizer { private static final int NO_TYPE = 0, NAME = 1, LONG_NAME = 2, SPECIAL = 3, NUMBER = 4, FLOAT = 5, STRING = 6, LONG = 7, DECIMAL = 8, BOOLEAN = 9, DATE = 10, TIME = 11, TIMESTAMP = 12, NULL = 13, NAMED_PARAM = 14; // used only internally private static final int QUOTED_IDENTIFIER = 15, REMARK_LINE = 16, REMARK = 17; private String sCommand; private int iLength; private int iIndex; private int tokenIndex; private int nextTokenIndex; private int beginIndex; private int iType; private String sToken; private int indexLongNameFirst = -1; private String sLongNameFirst = null; private int typeLongNameFirst; // getToken() will clear LongNameFirst unless retainFirst is set. private boolean retainFirst = false; // private String sLongNameLast; // WAIT. Don't do anything before popping another Token (because the // state variables aren't set properly due to a call of wait()). private boolean bWait; private boolean lastTokenQuotedID; // literals that are values static IntValueHashMap valueTokens; static { valueTokens = new IntValueHashMap(); valueTokens.put(Token.T_NULL, NULL); valueTokens.put(Token.T_TRUE, BOOLEAN); valueTokens.put(Token.T_FALSE, BOOLEAN); } public Tokenizer() {} public Tokenizer(String s) { sCommand = s; iLength = s.length(); iIndex = 0; } public void reset(String s) { sCommand = s; iLength = s.length(); iIndex = 0; tokenIndex = 0; nextTokenIndex = 0; beginIndex = 0; iType = NO_TYPE; typeLongNameFirst = NO_TYPE; sToken = null; indexLongNameFirst = -1; sLongNameFirst = null; // sLongNameLast = null; bWait = false; lastTokenQuotedID = false; retainFirst = false; } /** * * @throws HsqlException */ void back() throws HsqlException { if (bWait) { Trace.doAssert(false, "Querying state when in Wait mode"); } nextTokenIndex = iIndex; iIndex = ( indexLongNameFirst != -1 ) ? indexLongNameFirst : tokenIndex; bWait = true; } /** * get the given token or throw * * for commands and simple unquoted identifiers only * * @param match * * @throws HsqlException */ String getThis(String match) throws HsqlException { getToken(); matchThis(match); return sToken; } /** * for commands and simple unquoted identifiers only */ void matchThis(String match) throws HsqlException { if (bWait) { Trace.doAssert(false, "Querying state when in Wait mode"); } if (!sToken.equals(match) || iType == QUOTED_IDENTIFIER || iType == LONG_NAME) { String token = iType == LONG_NAME ? sLongNameFirst : sToken; throw Trace.error(Trace.UNEXPECTED_TOKEN, Trace.TOKEN_REQUIRED, new Object[] { token, match }); } } void throwUnexpected() throws HsqlException { String token = iType == LONG_NAME ? sLongNameFirst : sToken; throw Trace.error(Trace.UNEXPECTED_TOKEN, token); } /** * Used for commands only * * * @param match */ public boolean isGetThis(String match) throws HsqlException { getToken(); if (iType != QUOTED_IDENTIFIER && iType != LONG_NAME && sToken.equals(match)) { return true; } back(); return false; } /** * this methode is called before other wasXXX methods and takes * precedence */ boolean wasValue() throws HsqlException { if (bWait) { Trace.doAssert(false, "Querying state when in Wait mode"); } switch (iType) { case STRING : case NUMBER : case LONG : case FLOAT : case DECIMAL : case BOOLEAN : case NULL : return true; default : return false; } } boolean wasQuotedIdentifier() throws HsqlException { if (bWait) { Trace.doAssert(false, "Querying state when in Wait mode"); } return lastTokenQuotedID; // iType won't help for LONG_NAMEs. //return iType == QUOTED_IDENTIFIER; } boolean wasFirstQuotedIdentifier() throws HsqlException { if (bWait) { Trace.doAssert(false, "Querying state when in Wait mode"); } return (typeLongNameFirst == QUOTED_IDENTIFIER); } /** * Method declaration * * * @return */ boolean wasLongName() throws HsqlException { if (bWait) { Trace.doAssert(false, "Querying state when in Wait mode"); } return iType == LONG_NAME; } /** * Simple Name means a quoted or unquoted identifier without * qualifiers provided it is not in the hKeyword list. * * @return */ boolean wasSimpleName() throws HsqlException { if (bWait) { Trace.doAssert(false, "Querying state when in Wait mode"); } if (iType == QUOTED_IDENTIFIER && sToken.length() != 0) { return true; } if (iType != NAME) { return false; } return !Token.isKeyword(sToken); } /** * checks whether the previously obtained token was a (named) parameter * * @return true if the previously obtained token was a (named) parameter */ boolean wasParameter() throws HsqlException { Trace.doAssert(!bWait, "Querying state when in Wait mode"); return (iType == NAMED_PARAM); } /** * Name means all quoted and unquoted identifiers plus any word not in the * hKeyword list. * * @return true if it's a name */ boolean wasName() throws HsqlException { if (bWait) { Trace.doAssert(false, "Querying state when in Wait mode"); } if (iType == QUOTED_IDENTIFIER) { return true; } if (iType != NAME && iType != LONG_NAME) { return false; } return !Token.isKeyword(sToken); } String getLongNamePre() throws HsqlException { return null; } /** * Return first part of long name * * * @return */ String getLongNameFirst() throws HsqlException { if (bWait) { Trace.doAssert(false, "Querying state when in Wait mode"); } return sLongNameFirst; } boolean wasSimpleToken() throws HsqlException { return iType != QUOTED_IDENTIFIER && iType != LONG_NAME && iType != STRING && iType != NAMED_PARAM; } String getSimpleToken() throws HsqlException { getToken(); if (!wasSimpleToken()) { String token = iType == LONG_NAME ? sLongNameFirst : sToken; throw Trace.error(Trace.UNEXPECTED_TOKEN, token); } return sToken; } public boolean wasThis(String match) throws HsqlException { if (sToken.equals(match) && iType != QUOTED_IDENTIFIER && iType != LONG_NAME && iType != STRING) { return true; } return false; } /** * getName() is more broad than getSimpleName() in that it includes * 2-part names as well * * @return popped name * @throws HsqlException if next token is not an AName */ public String getName() throws HsqlException { getToken(); if (!wasName()) { throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken); } return sToken; } /** * Returns a single, unqualified name (identifier) * * @return name * @throws HsqlException */ public String getSimpleName() throws HsqlException { getToken(); if (!wasSimpleName()) { String token = iType == LONG_NAME ? sLongNameFirst : sToken; throw Trace.error(Trace.UNEXPECTED_TOKEN, token); } return sToken; } /** * Return any token. * * * @return * * @throws HsqlException */ public String getString() throws HsqlException { getToken(); return sToken; } int getInt() throws HsqlException { long v = getBigint(); if (v > Integer.MAX_VALUE || v < Integer.MIN_VALUE) { throw Trace.error(Trace.WRONG_DATA_TYPE, Types.getTypeString(getType())); } return (int) v; } static BigDecimal LONG_MAX_VALUE_INCREMENT = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.valueOf(1)); long getBigint() throws HsqlException { boolean minus = false; getToken(); if (sToken.equals("-")) { minus = true; getToken(); } Object o = getAsValue(); int t = getType(); switch (t) { case Types.INTEGER : case Types.BIGINT : break; case Types.DECIMAL : // only Long.MAX_VALUE + 1 together with minus is acceptable if (minus && LONG_MAX_VALUE_INCREMENT.equals(o)) { return Long.MIN_VALUE; } default : throw Trace.error(Trace.WRONG_DATA_TYPE, Types.getTypeString(t)); } long v = ((Number) o).longValue(); return minus ? -v : v; } Object getInType(int type) throws HsqlException { getToken(); Object o = getAsValue(); int t = getType(); if (t != type) { throw Trace.error(Trace.WRONG_DATA_TYPE, Types.getTypeString(t)); } return o; } /** * * * * @return */ public int getType() throws HsqlException { if (bWait) { Trace.doAssert(false, "Querying state when in Wait mode"); } // todo: make sure it's used only for Values! // todo: synchronize iType with hColumn switch (iType) { case STRING : return Types.VARCHAR; case NUMBER : return Types.INTEGER; case LONG : return Types.BIGINT; case FLOAT : return Types.DOUBLE; case DECIMAL : return Types.DECIMAL; case BOOLEAN : return Types.BOOLEAN; case DATE : return Types.DATE; case TIME : return Types.TIME; case TIMESTAMP : return Types.TIMESTAMP; default : return Types.NULL; } } /** * Method declaration * * * @return * * @throws HsqlException */ Object getAsValue() throws HsqlException { if (!wasValue()) { throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken); } switch (iType) { case NULL : return null; case STRING : //fredt - no longer returning string with a singlequote as last char return sToken; case LONG : return ValuePool.getLong(Long.parseLong(sToken)); case NUMBER : // fredt - this returns unsigned values which are later negated. // as a result Integer.MIN_VALUE or Long.MIN_VALUE are promoted // to a wider type. if (sToken.length() < 11) { try { return ValuePool.getInt(Integer.parseInt(sToken)); } catch (Exception e1) {} } if (sToken.length() < 20) { try { iType = LONG; return ValuePool.getLong(Long.parseLong(sToken)); } catch (Exception e2) {} } iType = DECIMAL; return new BigDecimal(sToken); case FLOAT : double d = JavaSystem.parseDouble(sToken); long l = Double.doubleToLongBits(d); return ValuePool.getDouble(l); case DECIMAL : return new BigDecimal(sToken); case BOOLEAN : return sToken.equalsIgnoreCase("TRUE") ? Boolean.TRUE : Boolean.FALSE; case DATE : return HsqlDateTime.dateValue(sToken); case TIME : return HsqlDateTime.timeValue(sToken); case TIMESTAMP : return HsqlDateTime.timestampValue(sToken); default : return sToken; } } /** * return the current position to be used for VIEW processing * * @return */ int getPosition() { return iIndex; } /** * mark the current position to be used for future getLastPart() calls * * @return */ String getPart(int begin, int end) { return sCommand.substring(begin, end); } /** * mark the current position to be used for future getLastPart() calls * * @return */ int getPartMarker() { return beginIndex; } /** * mark the current position to be used for future getLastPart() calls * */ void setPartMarker() { beginIndex = iIndex; } /** * mark the position to be used for future getLastPart() calls * */ void setPartMarker(int position) { beginIndex = position; } /** * return part of the command string from the last marked position * * @return */ String getLastPart() { return sCommand.substring(beginIndex, iIndex); } // fredt@users 20020910 - patch 1.7.1 by Nitin Chauhan - rewrite as switch /** * Method declaration * * * @throws HsqlException */ private void getToken() throws HsqlException { if (bWait) { bWait = false; iIndex = nextTokenIndex; return; } if (!retainFirst) { sLongNameFirst = null; indexLongNameFirst = -1; typeLongNameFirst = NO_TYPE; } while (iIndex < iLength && Character.isWhitespace(sCommand.charAt(iIndex))) { iIndex++; } sToken = ""; tokenIndex = iIndex; if (iIndex >= iLength) { iType = NO_TYPE; return; } char c = sCommand.charAt(iIndex); boolean point = false, digit = false, exp = false, afterexp = false; boolean end = false; char cfirst = 0; lastTokenQuotedID = false; if (Character.isJavaIdentifierStart(c)) { iType = NAME; } else if (Character.isDigit(c)) { iType = NUMBER; digit = true; } else { switch (c) { case '(' : sToken = Token.T_OPENBRACKET; iType = SPECIAL; iIndex++; return; case ')' : sToken = Token.T_CLOSEBRACKET; iType = SPECIAL; iIndex++; return; case ',' : sToken = Token.T_COMMA; iType = SPECIAL; iIndex++; return; case '*' : sToken = Token.T_MULTIPLY; iType = SPECIAL; iIndex++; return; case '=' : sToken = Token.T_EQUALS; iType = SPECIAL; iIndex++; return; case ';' : sToken = Token.T_SEMICOLON; iType = SPECIAL; iIndex++; return; case '+' : sToken = Token.T_PLUS; iType = SPECIAL; iIndex++; return; case '%' : sToken = Token.T_PERCENT; iType = SPECIAL; iIndex++; return; case '?' : sToken = Token.T_QUESTION; iType = SPECIAL; iIndex++; return; case ':' : Trace.check(++iIndex < iLength, Trace.UNEXPECTED_END_OF_COMMAND); c = sCommand.charAt(iIndex); Trace.check(Character.isJavaIdentifierStart(c), Trace.INVALID_IDENTIFIER, ":" + c); iType = NAMED_PARAM; break; case '\"' : lastTokenQuotedID = true; iType = QUOTED_IDENTIFIER; iIndex++; sToken = getString('"'); if (iIndex == sCommand.length()) { return; } c = sCommand.charAt(iIndex); if (c == '.') { sLongNameFirst = sToken; indexLongNameFirst = tokenIndex; typeLongNameFirst = iType; iIndex++; if (retainFirst) { throw Trace.error(Trace.THREE_PART_IDENTIFIER); } // fredt - todo - avoid recursion - this has problems when there is whitespace // after the dot - the same with NAME retainFirst = true; getToken(); retainFirst = false; iType = LONG_NAME; } return; case '\'' : iType = STRING; iIndex++; sToken = getString('\''); return; case '!' : case '<' : case '>' : case '|' : case '/' : case '-' : cfirst = c; iType = SPECIAL; break; case '.' : iType = DECIMAL; point = true; break; default : throw Trace.error(Trace.UNEXPECTED_TOKEN, String.valueOf(c)); } } int start = iIndex++; while (true) { if (iIndex >= iLength) { c = ' '; end = true; Trace.check(iType != STRING && iType != QUOTED_IDENTIFIER, Trace.UNEXPECTED_END_OF_COMMAND); } else { c = sCommand.charAt(iIndex); } switch (iType) { case NAMED_PARAM : case NAME : if (Character.isJavaIdentifierPart(c)) { break; } // fredt - todo new char[] to back sToken sToken = sCommand.substring(start, iIndex).toUpperCase(Locale.ENGLISH); // the following only for NAME, not for NAMED_PARAM if (iType == NAMED_PARAM) { return; } if (c == '.') { typeLongNameFirst = iType; sLongNameFirst = sToken; indexLongNameFirst = tokenIndex; iIndex++; if (retainFirst) { throw Trace.error(Trace.THREE_PART_IDENTIFIER); } retainFirst = true; getToken(); // todo: eliminate recursion retainFirst = false; iType = LONG_NAME; } else if (c == '(') { // it is a function call } else { // if in value list then it is a value int type = valueTokens.get(sToken, -1); if (type != -1) { iType = type; } } return; case QUOTED_IDENTIFIER : case STRING : // shouldn't get here break; case REMARK : if (end) { // unfinished remark // maybe print error here iType = NO_TYPE; return; } else if (c == '*') { iIndex++; if (iIndex < iLength && sCommand.charAt(iIndex) == '/') { // using recursion here iIndex++; getToken(); return; } } break; case REMARK_LINE : if (end) { iType = NO_TYPE; return; } else if (c == '\r' || c == '\n') { // using recursion here getToken(); return; } break; case SPECIAL : if (c == '/' && cfirst == '/') { iType = REMARK_LINE; break; } else if (c == '-' && cfirst == '-') { iType = REMARK_LINE; break; } else if (c == '*' && cfirst == '/') { iType = REMARK; break; } else if (c == '>' || c == '=' || c == '|') { break; } sToken = sCommand.substring(start, iIndex); return; case NUMBER : case FLOAT : case DECIMAL : if (Character.isDigit(c)) { digit = true; } else if (c == '.') { iType = DECIMAL; if (point) { throw Trace.error(Trace.UNEXPECTED_TOKEN, "."); } point = true; } else if (c == 'E' || c == 'e') { if (exp) { throw Trace.error(Trace.UNEXPECTED_TOKEN, "E"); } // HJB-2001-08-2001 - now we are sure it's a float iType = FLOAT; // first character after exp may be + or - afterexp = true; point = true; exp = true; } else if (c == '-' && afterexp) { afterexp = false; } else if (c == '+' && afterexp) { afterexp = false; } else { afterexp = false; if (!digit) { if (point && start == iIndex - 1) { sToken = "."; iType = SPECIAL; return; } throw Trace.error(Trace.UNEXPECTED_TOKEN, String.valueOf(c)); } sToken = sCommand.substring(start, iIndex); return; } } iIndex++; } } // fredt - strings are constructed from new char[] objects to avoid slack // because these strings might end up as part of internal data structures // or table elements. // we may consider using pools to avoid recreating the strings private String getString(char quoteChar) throws HsqlException { try { int nextIndex = iIndex; boolean quoteInside = false; for (;;) { nextIndex = sCommand.indexOf(quoteChar, nextIndex); if (nextIndex < 0) { throw Trace.error(Trace.UNEXPECTED_END_OF_COMMAND); } if (nextIndex < iLength - 1 && sCommand.charAt(nextIndex + 1) == quoteChar) { quoteInside = true; nextIndex += 2; continue; } break; } char[] chBuffer = new char[nextIndex - iIndex]; sCommand.getChars(iIndex, nextIndex, chBuffer, 0); int j = chBuffer.length; if (quoteInside) { j = 0; // fredt - loop assumes all occurences of quoteChar are paired // this has already been checked by the preprocessing loop for (int i = 0; i < chBuffer.length; i++, j++) { if (chBuffer[i] == quoteChar) { i++; } chBuffer[j] = chBuffer[i]; } } iIndex = ++nextIndex; return new String(chBuffer, 0, j); } catch (HsqlException e) { throw e; } catch (Exception e) { e.toString(); } return null; } /** * Method declaration * * * @return */ int getLength() { return iLength; } } Other HSQLDB examples (source code examples)Here is a short list of links related to this HSQLDB Tokenizer.java source code file: |
... this post is sponsored by my books ... | |
#1 New Release! |
FP Best Seller |
Copyright 1998-2024 Alvin Alexander, alvinalexander.com
All Rights Reserved.
A percentage of advertising revenue from
pages under the /java/jwarehouse
URI on this website is
paid back to open source projects.