|
What this is
Other links
The source code/* * ASPPerlscriptTokenMarker.java - Perlscript token marker * Copyright (C) 1999 Andr? Kaplan * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. */ package org.gjt.sp.jedit.syntax; import javax.swing.text.Segment; /** * Original Perl token marker by Slava Pestov * Perlscript Token Marker * * @author Andre Kaplan * @version 0.6 */ public class ASPPerlscriptTokenMarker extends TokenMarker implements TokenMarkerWithAddToken, MultiModeTokenMarkerWithContext { // public members public static final byte S_ONE = Token.INTERNAL_FIRST; public static final byte S_TWO = (byte)(Token.INTERNAL_FIRST + 1); public static final byte S_END = (byte)(Token.INTERNAL_FIRST + 2); public ASPPerlscriptTokenMarker() { this(getKeywords(), true); } public ASPPerlscriptTokenMarker(boolean standalone) { this(getKeywords(), standalone); } public ASPPerlscriptTokenMarker(KeywordMap keywords) { this(keywords, true); } public ASPPerlscriptTokenMarker(KeywordMap keywords, boolean standalone) { this.keywords = keywords; this.standalone = standalone; } public void addToken(int length, byte id) { super.addToken(length, id); } protected byte markTokensImpl(byte _token, Segment line, int lineIndex) { TokenMarkerContext tokenContext = new TokenMarkerContext(line, lineIndex, this, this.lineInfo); MultiModeToken prevLineToken = MultiModeToken.NULL; if ( tokenContext.prevLineInfo != null && tokenContext.prevLineInfo.obj != null && tokenContext.prevLineInfo.obj instanceof MultiModeToken ) { prevLineToken = (MultiModeToken)tokenContext.prevLineInfo.obj; } MultiModeToken res = this.markTokensImpl(prevLineToken, tokenContext); tokenContext.currLineInfo.obj = res; return res.token; } // ************** // markTokensImpl // ************** public MultiModeToken markTokensImpl(final MultiModeToken token, TokenMarkerContext tokenContext) { MultiModeToken res = new MultiModeToken(token); matchChar = '\0'; matchCharBracket = false; matchSpacesAllowed = false; int debugPos = -1; int debugCount = 0; if ( res.token == Token.LITERAL1 && res.obj != null) { String str = (String)res.obj; if ( str != null && str.length() == tokenContext.line.count && tokenContext.regionMatches(false, str)) { tokenContext.addTokenToEnd(res.token); res.token = Token.NULL; res.obj = null; return res; } else { tokenContext.addTokenToEnd(res.token); return res; } } boolean backslash = false; loop: for(this.debug.reset(); tokenContext.hasMoreChars(); ) { char c = tokenContext.getChar(); // Following is a way to detect whether tokenContext.pos is not // correctly incremented. This is for debugging purposes if (!this.debug.isOK(tokenContext)) { // We got stuck here at some point // Log this and increment tokenContext.pos to escape this tokenContext.pos++; } if (c == '\\') { backslash = !backslash; tokenContext.pos++; continue; } switch (res.token) { case Token.NULL: if (!this.standalone) { if (res.mode == ASPMode.CSPS) { if (tokenContext.regionMatches(true, "<%")) { this.doKeywordToPos(res,tokenContext,c); // Just return, let parent mode decide what to do next return res; } } if (res.mode == ASPMode.ASP) { if (tokenContext.regionMatches(true, "%>")) { this.doKeywordToPos(res,tokenContext,c); // Just return, let parent mode decide what to do next return res; } } if (res.mode == ASPMode.CSPS || res.mode == ASPMode.SSPS) { if (tokenContext.regionMatches(true, "")) { this.doKeywordToPos(res,tokenContext,c); // Just return, let parent mode decide what to do next return res; } } } switch(c) { case '#': if (this.doKeywordToPos(res,tokenContext,c)) break; if (backslash) { backslash = false; } else { tokenContext.addTokenToPos(res.token); tokenContext.addTokenToEnd(Token.COMMENT1); break loop; } break; case '=': backslash = false; if (tokenContext.atFirst()) { res.token = Token.COMMENT2; tokenContext.addTokenToEnd(res.token); break loop; } else this.doKeywordToPos(res,tokenContext,c); break; case '$': case '&': case '%': case '@': backslash = false; if(this.doKeywordToPos(res,tokenContext,c)) break; if (tokenContext.remainingChars() > 0) { char c1 = tokenContext.getChar(1); if ( (c == '&') && ( (c1 == '&') || (Character.isWhitespace(c1)) ) ) { tokenContext.pos++; } else { tokenContext.addTokenToPos(res.token); res.token = Token.KEYWORD2; } } break; case '"': if(this.doKeywordToPos(res,tokenContext,c)) break; if(backslash) { backslash = false; } else { tokenContext.addTokenToPos(res.token); res.token = Token.LITERAL1; res.obj = null; } break; case '\'': if(backslash) { backslash = false; } else { int oldLastKeyword = tokenContext.lastKeyword; if(this.doKeywordToPos(res,tokenContext,c)) break; if (tokenContext.pos != oldLastKeyword) break; tokenContext.addTokenToPos(res.token); res.token = Token.LITERAL2; } break; case '`': if(this.doKeywordToPos(res,tokenContext,c)) break; if(backslash) { backslash = false; } else { tokenContext.addTokenToPos(res.token); res.token = Token.OPERATOR; } break; case '<': if(this.doKeywordToPos(res,tokenContext,c)) break; if(backslash) backslash = false; else { if ( (tokenContext.remainingChars() > 1) && (tokenContext.getChar(1) == '<') && (!Character.isWhitespace( tokenContext.getChar(2))) ) { tokenContext.addTokenToPos(res.token); res.token = Token.LITERAL1; int len = tokenContext.remainingChars() - 1; if(tokenContext.lastChar() == ';') len--; String readin = createReadinString( tokenContext.array, tokenContext.pos + 2, len ); // Log.log(Log.DEBUG, this, "Readin: [" + readin + "]"); res.obj = readin; tokenContext.addTokenToEnd(res.token); break loop; } } break; case ':': backslash = false; if(this.doKeywordToPos(res,tokenContext,c)) break; // Doesn't pick up all labels, // but at least doesn't mess up // XXX::YYY if(tokenContext.lastKeyword != 0) break; tokenContext.pos++; tokenContext.addTokenToPos(Token.LABEL); continue; case '-': backslash = false; if(this.doKeywordToPos(res,tokenContext,c)) break; if( (tokenContext.pos != tokenContext.lastKeyword) || (tokenContext.remainingChars() < 1) ) break; switch(tokenContext.getChar(1)) { case 'r': case 'w': case 'x': case 'o': case 'R': case 'W': case 'X': case 'O': case 'e': case 'z': case 's': case 'f': case 'd': case 'l': case 'p': case 'S': case 'b': case 'c': case 't': case 'u': case 'g': case 'k': case 'T': case 'B': case 'M': case 'A': case 'C': tokenContext.addTokenToPos(res.token); tokenContext.pos++; tokenContext.addTokenToPos(Token.KEYWORD3); tokenContext.pos++; continue; } break; case '/': case '?': if(this.doKeywordToPos(res,tokenContext,c)) break; if(tokenContext.remainingChars() > 0) { backslash = false; char ch = tokenContext.getChar(1); if(Character.isWhitespace(ch)) break; matchChar = c; matchSpacesAllowed = false; tokenContext.addTokenToPos(res.token); res.token = S_ONE; } break; default: backslash = false; if(!Character.isLetterOrDigit(c) && c != '_') this.doKeywordToPos(res,tokenContext,c); break; } break; case Token.KEYWORD2: backslash = false; // This test checks for an end-of-variable // condition if(!Character.isLetterOrDigit(c) && c != '_' && c != '#' && c != '\'' && c != ':' && c != '&') { // If this is the first character // of the variable name ($'aaa) // ignore it if (!tokenContext.atFirst() && tokenContext.getChar(-1) == '$') { tokenContext.pos++; tokenContext.addTokenToPos(res.token); continue; } // Otherwise, end of variable... else { tokenContext.addTokenToPos(res.token); // Wind back so that stuff // like $hello$fred is picked // up // Change the token KEYWORD2 -> NULL // And continue at the same position res.token = Token.NULL; continue; } } break; case S_ONE: case S_TWO: if(backslash) backslash = false; else { if(matchChar == '\0') { if(Character.isWhitespace(matchChar) && !matchSpacesAllowed) break; else matchChar = c; } else { switch(matchChar) { case '(': matchChar = ')'; matchCharBracket = true; break; case '[': matchChar = ']'; matchCharBracket = true; break; case '{': matchChar = '}'; matchCharBracket = true; break; case '<': matchChar = '>'; matchCharBracket = true; break; default: matchCharBracket = false; break; } if(c != matchChar) break; if (res.token == S_TWO) { res.token = S_ONE; if(matchCharBracket) matchChar = '\0'; } else { res.token = S_END; tokenContext.pos++; tokenContext.addTokenToPos(Token.LITERAL2); continue; } } } break; case S_END: backslash = false; if(!Character.isLetterOrDigit(c) && c != '_') this.doKeywordToPos(res,tokenContext,c); break; case Token.COMMENT2: backslash = false; if (tokenContext.atFirst()) { if (tokenContext.regionMatches(false, "=cut")) res.token = Token.NULL; tokenContext.addTokenToEnd(Token.COMMENT2); break loop; } break; case Token.LITERAL1: if(backslash) backslash = false; /* else if(c == '$') backslash = true; */ else if (c == '"') { tokenContext.pos++; tokenContext.addTokenToPos(res.token); res.token = Token.NULL; continue; } break; case Token.LITERAL2: if(backslash) backslash = false; /* else if(c == '$') backslash = true; */ else if(c == '\'') { tokenContext.pos++; tokenContext.addTokenToPos(Token.LITERAL1); res.token = Token.NULL; continue; } break; case Token.OPERATOR: if(backslash) backslash = false; else if (c == '`') { tokenContext.pos++; tokenContext.addTokenToPos(res.token); res.token = Token.NULL; continue; } break; default: throw new InternalError("Invalid state: " + res.token); } tokenContext.pos++; } if(res.token == Token.NULL) this.doKeywordToEnd(res, tokenContext, '\0'); switch (res.token) { case Token.KEYWORD2: tokenContext.addTokenToEnd(res.token); break; case Token.LITERAL2: tokenContext.addTokenToEnd(Token.LITERAL1); break; case S_END: tokenContext.addTokenToEnd(Token.LITERAL2); res.token = Token.NULL; break; case S_ONE: case S_TWO: tokenContext.addTokenToEnd(Token.INVALID); // XXX res.token = Token.NULL; break; default: tokenContext.addTokenToEnd(res.token); break; } return res; } // private members private KeywordMap keywords; private boolean standalone; private char matchChar; private boolean matchCharBracket; private boolean matchSpacesAllowed; private TokenMarkerDebugger debug = new TokenMarkerDebugger(); // ************** // doKeywordToEnd // ************** private boolean doKeywordToEnd(MultiModeToken token, TokenMarkerContext tokenContext, char c) { return doKeyword(token, tokenContext, tokenContext.length, c); } // ************** // doKeywordToPos // ************** private boolean doKeywordToPos(MultiModeToken token, TokenMarkerContext tokenContext, char c) { return doKeyword(token, tokenContext, tokenContext.pos, c); } // ********* // doKeyword // ********* private boolean doKeyword(MultiModeToken token, TokenMarkerContext tokenContext, int i, char c) { int i1 = i + 1; if (token.token == S_END) { tokenContext.addTokenToPos(i, Token.LITERAL2); token.token = Token.NULL; tokenContext.lastKeyword = i1; return false; } int len = i - tokenContext.lastKeyword; byte id = keywords.lookup(tokenContext.line,tokenContext.lastKeyword,len); if (id == S_ONE || id == S_TWO) { tokenContext.addTokenToPos(tokenContext.lastKeyword, Token.NULL); tokenContext.addTokenToPos(i, Token.LITERAL2); tokenContext.lastKeyword = i1; if(Character.isWhitespace(c)) matchChar = '\0'; else matchChar = c; matchSpacesAllowed = true; token.token = id; return true; } else if (id != Token.NULL) { tokenContext.addTokenToPos(tokenContext.lastKeyword, Token.NULL); tokenContext.addTokenToPos(i, id); } tokenContext.lastKeyword = i1; return false; } // Converts < EOF >, < 'EOF' >, etc to |
... this post is sponsored by my books ... | |
#1 New Release! |
FP Best Seller |
Copyright 1998-2024 Alvin Alexander, alvinalexander.com
All Rights Reserved.
A percentage of advertising revenue from
pages under the /java/jwarehouse
URI on this website is
paid back to open source projects.