|
What this is
Other links
The source code
/*
* ASPPerlscriptTokenMarker.java - Perlscript token marker
* Copyright (C) 1999 Andr? Kaplan
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
*/
package org.gjt.sp.jedit.syntax;
import javax.swing.text.Segment;
/**
* Original Perl token marker by Slava Pestov
* Perlscript Token Marker
*
* @author Andre Kaplan
* @version 0.6
*/
public class ASPPerlscriptTokenMarker
extends TokenMarker
implements TokenMarkerWithAddToken,
MultiModeTokenMarkerWithContext
{
// public members
public static final byte S_ONE = Token.INTERNAL_FIRST;
public static final byte S_TWO = (byte)(Token.INTERNAL_FIRST + 1);
public static final byte S_END = (byte)(Token.INTERNAL_FIRST + 2);
public ASPPerlscriptTokenMarker()
{
this(getKeywords(), true);
}
public ASPPerlscriptTokenMarker(boolean standalone)
{
this(getKeywords(), standalone);
}
public ASPPerlscriptTokenMarker(KeywordMap keywords)
{
this(keywords, true);
}
public ASPPerlscriptTokenMarker(KeywordMap keywords, boolean standalone)
{
this.keywords = keywords;
this.standalone = standalone;
}
public void addToken(int length, byte id)
{
super.addToken(length, id);
}
protected byte markTokensImpl(byte _token, Segment line, int lineIndex)
{
TokenMarkerContext tokenContext = new TokenMarkerContext(line, lineIndex, this, this.lineInfo);
MultiModeToken prevLineToken = MultiModeToken.NULL;
if ( tokenContext.prevLineInfo != null
&& tokenContext.prevLineInfo.obj != null
&& tokenContext.prevLineInfo.obj instanceof MultiModeToken
)
{
prevLineToken = (MultiModeToken)tokenContext.prevLineInfo.obj;
}
MultiModeToken res = this.markTokensImpl(prevLineToken, tokenContext);
tokenContext.currLineInfo.obj = res;
return res.token;
}
// **************
// markTokensImpl
// **************
public MultiModeToken markTokensImpl(final MultiModeToken token, TokenMarkerContext tokenContext)
{
MultiModeToken res = new MultiModeToken(token);
matchChar = '\0';
matchCharBracket = false;
matchSpacesAllowed = false;
int debugPos = -1;
int debugCount = 0;
if ( res.token == Token.LITERAL1
&& res.obj != null)
{
String str = (String)res.obj;
if ( str != null
&& str.length() == tokenContext.line.count
&& tokenContext.regionMatches(false, str))
{
tokenContext.addTokenToEnd(res.token);
res.token = Token.NULL;
res.obj = null;
return res;
} else {
tokenContext.addTokenToEnd(res.token);
return res;
}
}
boolean backslash = false;
loop: for(this.debug.reset(); tokenContext.hasMoreChars(); )
{
char c = tokenContext.getChar();
// Following is a way to detect whether tokenContext.pos is not
// correctly incremented. This is for debugging purposes
if (!this.debug.isOK(tokenContext)) {
// We got stuck here at some point
// Log this and increment tokenContext.pos to escape this
tokenContext.pos++;
}
if (c == '\\')
{
backslash = !backslash;
tokenContext.pos++;
continue;
}
switch (res.token)
{
case Token.NULL:
if (!this.standalone) {
if (res.mode == ASPMode.CSPS) {
if (tokenContext.regionMatches(true, "<%"))
{
this.doKeywordToPos(res,tokenContext,c);
// Just return, let parent mode decide what to do next
return res;
}
}
if (res.mode == ASPMode.ASP) {
if (tokenContext.regionMatches(true, "%>"))
{
this.doKeywordToPos(res,tokenContext,c);
// Just return, let parent mode decide what to do next
return res;
}
}
if (res.mode == ASPMode.CSPS || res.mode == ASPMode.SSPS) {
if (tokenContext.regionMatches(true, ""))
{
this.doKeywordToPos(res,tokenContext,c);
// Just return, let parent mode decide what to do next
return res;
}
}
}
switch(c)
{
case '#':
if (this.doKeywordToPos(res,tokenContext,c))
break;
if (backslash) {
backslash = false;
} else {
tokenContext.addTokenToPos(res.token);
tokenContext.addTokenToEnd(Token.COMMENT1);
break loop;
}
break;
case '=':
backslash = false;
if (tokenContext.atFirst())
{
res.token = Token.COMMENT2;
tokenContext.addTokenToEnd(res.token);
break loop;
}
else
this.doKeywordToPos(res,tokenContext,c);
break;
case '$': case '&': case '%': case '@':
backslash = false;
if(this.doKeywordToPos(res,tokenContext,c))
break;
if (tokenContext.remainingChars() > 0)
{
char c1 = tokenContext.getChar(1);
if ( (c == '&')
&& ( (c1 == '&')
|| (Character.isWhitespace(c1))
)
)
{
tokenContext.pos++;
} else {
tokenContext.addTokenToPos(res.token);
res.token = Token.KEYWORD2;
}
}
break;
case '"':
if(this.doKeywordToPos(res,tokenContext,c))
break;
if(backslash) {
backslash = false;
} else {
tokenContext.addTokenToPos(res.token);
res.token = Token.LITERAL1;
res.obj = null;
}
break;
case '\'':
if(backslash) {
backslash = false;
} else {
int oldLastKeyword = tokenContext.lastKeyword;
if(this.doKeywordToPos(res,tokenContext,c))
break;
if (tokenContext.pos != oldLastKeyword)
break;
tokenContext.addTokenToPos(res.token);
res.token = Token.LITERAL2;
}
break;
case '`':
if(this.doKeywordToPos(res,tokenContext,c))
break;
if(backslash) {
backslash = false;
} else {
tokenContext.addTokenToPos(res.token);
res.token = Token.OPERATOR;
}
break;
case '<':
if(this.doKeywordToPos(res,tokenContext,c))
break;
if(backslash)
backslash = false;
else
{
if ( (tokenContext.remainingChars() > 1)
&& (tokenContext.getChar(1) == '<')
&& (!Character.isWhitespace(
tokenContext.getChar(2)))
)
{
tokenContext.addTokenToPos(res.token);
res.token = Token.LITERAL1;
int len = tokenContext.remainingChars() - 1;
if(tokenContext.lastChar() == ';')
len--;
String readin =
createReadinString(
tokenContext.array,
tokenContext.pos + 2,
len
);
// Log.log(Log.DEBUG, this, "Readin: [" + readin + "]");
res.obj = readin;
tokenContext.addTokenToEnd(res.token);
break loop;
}
}
break;
case ':':
backslash = false;
if(this.doKeywordToPos(res,tokenContext,c))
break;
// Doesn't pick up all labels,
// but at least doesn't mess up
// XXX::YYY
if(tokenContext.lastKeyword != 0)
break;
tokenContext.pos++;
tokenContext.addTokenToPos(Token.LABEL);
continue;
case '-':
backslash = false;
if(this.doKeywordToPos(res,tokenContext,c))
break;
if( (tokenContext.pos != tokenContext.lastKeyword)
|| (tokenContext.remainingChars() < 1)
)
break;
switch(tokenContext.getChar(1))
{
case 'r': case 'w': case 'x':
case 'o': case 'R': case 'W':
case 'X': case 'O': case 'e':
case 'z': case 's': case 'f':
case 'd': case 'l': case 'p':
case 'S': case 'b': case 'c':
case 't': case 'u': case 'g':
case 'k': case 'T': case 'B':
case 'M': case 'A': case 'C':
tokenContext.addTokenToPos(res.token);
tokenContext.pos++;
tokenContext.addTokenToPos(Token.KEYWORD3);
tokenContext.pos++;
continue;
}
break;
case '/': case '?':
if(this.doKeywordToPos(res,tokenContext,c))
break;
if(tokenContext.remainingChars() > 0)
{
backslash = false;
char ch = tokenContext.getChar(1);
if(Character.isWhitespace(ch))
break;
matchChar = c;
matchSpacesAllowed = false;
tokenContext.addTokenToPos(res.token);
res.token = S_ONE;
}
break;
default:
backslash = false;
if(!Character.isLetterOrDigit(c)
&& c != '_')
this.doKeywordToPos(res,tokenContext,c);
break;
}
break;
case Token.KEYWORD2:
backslash = false;
// This test checks for an end-of-variable
// condition
if(!Character.isLetterOrDigit(c) && c != '_'
&& c != '#' && c != '\'' && c != ':'
&& c != '&')
{
// If this is the first character
// of the variable name ($'aaa)
// ignore it
if (!tokenContext.atFirst() && tokenContext.getChar(-1) == '$')
{
tokenContext.pos++;
tokenContext.addTokenToPos(res.token);
continue;
}
// Otherwise, end of variable...
else
{
tokenContext.addTokenToPos(res.token);
// Wind back so that stuff
// like $hello$fred is picked
// up
// Change the token KEYWORD2 -> NULL
// And continue at the same position
res.token = Token.NULL;
continue;
}
}
break;
case S_ONE: case S_TWO:
if(backslash)
backslash = false;
else
{
if(matchChar == '\0')
{
if(Character.isWhitespace(matchChar)
&& !matchSpacesAllowed)
break;
else
matchChar = c;
}
else
{
switch(matchChar)
{
case '(':
matchChar = ')';
matchCharBracket = true;
break;
case '[':
matchChar = ']';
matchCharBracket = true;
break;
case '{':
matchChar = '}';
matchCharBracket = true;
break;
case '<':
matchChar = '>';
matchCharBracket = true;
break;
default:
matchCharBracket = false;
break;
}
if(c != matchChar)
break;
if (res.token == S_TWO)
{
res.token = S_ONE;
if(matchCharBracket)
matchChar = '\0';
}
else
{
res.token = S_END;
tokenContext.pos++;
tokenContext.addTokenToPos(Token.LITERAL2);
continue;
}
}
}
break;
case S_END:
backslash = false;
if(!Character.isLetterOrDigit(c)
&& c != '_')
this.doKeywordToPos(res,tokenContext,c);
break;
case Token.COMMENT2:
backslash = false;
if (tokenContext.atFirst())
{
if (tokenContext.regionMatches(false, "=cut"))
res.token = Token.NULL;
tokenContext.addTokenToEnd(Token.COMMENT2);
break loop;
}
break;
case Token.LITERAL1:
if(backslash)
backslash = false;
/* else if(c == '$')
backslash = true; */
else if (c == '"')
{
tokenContext.pos++;
tokenContext.addTokenToPos(res.token);
res.token = Token.NULL;
continue;
}
break;
case Token.LITERAL2:
if(backslash)
backslash = false;
/* else if(c == '$')
backslash = true; */
else if(c == '\'')
{
tokenContext.pos++;
tokenContext.addTokenToPos(Token.LITERAL1);
res.token = Token.NULL;
continue;
}
break;
case Token.OPERATOR:
if(backslash)
backslash = false;
else if (c == '`')
{
tokenContext.pos++;
tokenContext.addTokenToPos(res.token);
res.token = Token.NULL;
continue;
}
break;
default:
throw new InternalError("Invalid state: "
+ res.token);
}
tokenContext.pos++;
}
if(res.token == Token.NULL)
this.doKeywordToEnd(res, tokenContext, '\0');
switch (res.token)
{
case Token.KEYWORD2:
tokenContext.addTokenToEnd(res.token);
break;
case Token.LITERAL2:
tokenContext.addTokenToEnd(Token.LITERAL1);
break;
case S_END:
tokenContext.addTokenToEnd(Token.LITERAL2);
res.token = Token.NULL;
break;
case S_ONE: case S_TWO:
tokenContext.addTokenToEnd(Token.INVALID); // XXX
res.token = Token.NULL;
break;
default:
tokenContext.addTokenToEnd(res.token);
break;
}
return res;
}
// private members
private KeywordMap keywords;
private boolean standalone;
private char matchChar;
private boolean matchCharBracket;
private boolean matchSpacesAllowed;
private TokenMarkerDebugger debug = new TokenMarkerDebugger();
// **************
// doKeywordToEnd
// **************
private boolean doKeywordToEnd(MultiModeToken token,
TokenMarkerContext tokenContext,
char c)
{
return doKeyword(token, tokenContext, tokenContext.length, c);
}
// **************
// doKeywordToPos
// **************
private boolean doKeywordToPos(MultiModeToken token,
TokenMarkerContext tokenContext,
char c)
{
return doKeyword(token, tokenContext, tokenContext.pos, c);
}
// *********
// doKeyword
// *********
private boolean doKeyword(MultiModeToken token,
TokenMarkerContext tokenContext,
int i,
char c)
{
int i1 = i + 1;
if (token.token == S_END)
{
tokenContext.addTokenToPos(i, Token.LITERAL2);
token.token = Token.NULL;
tokenContext.lastKeyword = i1;
return false;
}
int len = i - tokenContext.lastKeyword;
byte id = keywords.lookup(tokenContext.line,tokenContext.lastKeyword,len);
if (id == S_ONE || id == S_TWO)
{
tokenContext.addTokenToPos(tokenContext.lastKeyword, Token.NULL);
tokenContext.addTokenToPos(i, Token.LITERAL2);
tokenContext.lastKeyword = i1;
if(Character.isWhitespace(c))
matchChar = '\0';
else
matchChar = c;
matchSpacesAllowed = true;
token.token = id;
return true;
}
else if (id != Token.NULL)
{
tokenContext.addTokenToPos(tokenContext.lastKeyword, Token.NULL);
tokenContext.addTokenToPos(i, id);
}
tokenContext.lastKeyword = i1;
return false;
}
// Converts < EOF >, < 'EOF' >, etc to
|
| ... this post is sponsored by my books ... | |
#1 New Release! |
FP Best Seller |
Copyright 1998-2024 Alvin Alexander, alvinalexander.com
All Rights Reserved.
A percentage of advertising revenue from
pages under the /java/jwarehouse
URI on this website is
paid back to open source projects.