/*
* Copyright (c) 2012, the Dart project authors.
*
* Licensed under the Eclipse Public License v1.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.eclipse.org/legal/epl-v10.html
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package com.google.dart.engine.scanner;
import com.google.dart.engine.error.AnalysisError;
import com.google.dart.engine.error.AnalysisErrorListener;
import com.google.dart.engine.source.Source;
import com.google.dart.engine.utilities.collection.IntList;
import com.google.dart.engine.utilities.instrumentation.Instrumentation;
import com.google.dart.engine.utilities.instrumentation.InstrumentationBuilder;
import java.util.ArrayList;
import java.util.List;
/**
* The class {@code Scanner} implements a scanner for Dart code.
* <p>
* The lexical structure of Dart is ambiguous without knowledge of the context in which a token is
* being scanned. For example, without context we cannot determine whether source of the form "<<"
* should be scanned as a single left-shift operator or as two left angle brackets. This scanner
* does not have any context, so it always resolves such conflicts by scanning the longest possible
* token.
*
* @coverage dart.engine.parser
*/
public class Scanner {
/**
* The source being scanned.
*/
private final Source source;
/**
* The reader used to access the characters in the source.
*/
private CharacterReader reader;
/**
* The error listener that will be informed of any errors that are found during the scan.
*/
private AnalysisErrorListener errorListener;
/**
* The flag specifying if documentation comments should be parsed.
*/
private boolean preserveComments = true;
/**
* The token pointing to the head of the linked list of tokens.
*/
private final Token tokens;
/**
* The last token that was scanned.
*/
private Token tail;
/**
* The first token in the list of comment tokens found since the last non-comment token.
*/
private Token firstComment;
/**
* The last token in the list of comment tokens found since the last non-comment token.
*/
private Token lastComment;
/**
* The index of the first character of the current token.
*/
private int tokenStart;
/**
* A list containing the offsets of the first character of each line in the source code.
*/
private IntList lineStarts = new IntList(1024);
/**
* A list, treated something like a stack, of tokens representing the beginning of a matched pair.
* It is used to pair the end tokens with the begin tokens.
*/
private List<BeginToken> groupingStack = new ArrayList<BeginToken>(128);
/**
* The index of the last item in the {@link #groupingStack}, or {@code -1} if the stack is empty.
*/
private int stackEnd = -1;
/**
* A flag indicating whether any unmatched groups were found during the parse.
*/
private boolean hasUnmatchedGroups = false;
/**
* Initialize a newly created scanner.
*
* @param source the source being scanned
* @param reader the character reader used to read the characters in the source
* @param errorListener the error listener that will be informed of any errors that are found
*/
public Scanner(Source source, CharacterReader reader, AnalysisErrorListener errorListener) {
this.source = source;
this.reader = reader;
this.errorListener = errorListener;
tokens = new Token(TokenType.EOF, -1);
tokens.setNext(tokens);
tail = tokens;
tokenStart = -1;
lineStarts.add(0);
}
/**
* Return an array containing the offsets of the first character of each line in the source code.
*
* @return an array containing the offsets of the first character of each line in the source code
*/
public int[] getLineStarts() {
return lineStarts.toArray();
}
/**
* Return {@code true} if any unmatched groups were found during the parse.
*
* @return {@code true} if any unmatched groups were found during the parse
*/
public boolean hasUnmatchedGroups() {
return hasUnmatchedGroups;
}
/**
* Set whether documentation tokens should be scanned.
*
* @param preserveComments {@code true} if documentation tokens should be scanned
*/
public void setPreserveComments(boolean preserveComments) {
this.preserveComments = preserveComments;
}
/**
* Record that the source begins on the given line and column at the current offset as given by
* the reader. The line starts for lines before the given line will not be correct.
* <p>
* This method must be invoked at most one time and must be invoked before scanning begins. The
* values provided must be sensible. The results are undefined if these conditions are violated.
*
* @param line the one-based index of the line containing the first character of the source
* @param column the one-based index of the column in which the first character of the source
* occurs
*/
public void setSourceStart(int line, int column) {
int offset = reader.getOffset();
if (line < 1 || column < 1 || offset < 0 || (line + column - 2) >= offset) {
return;
}
for (int i = 2; i < line; i++) {
lineStarts.add(1);
}
lineStarts.add(offset - column + 1);
}
/**
* Scan the source code to produce a list of tokens representing the source.
*
* @return the first token in the list of tokens that were produced
*/
public Token tokenize() {
InstrumentationBuilder instrumentation = Instrumentation.builder("dart.engine.AbstractScanner.tokenize");
int tokenCounter = 0;
try {
int next = reader.advance();
while (next != -1) {
tokenCounter++;
next = bigSwitch(next);
}
appendEofToken();
instrumentation.metric("tokensCount", tokenCounter);
return getFirstToken();
} finally {
instrumentation.log(2); //Log if over 1ms
}
}
/**
* Append the given token to the end of the token stream being scanned. This method is intended to
* be used by subclasses that copy existing tokens and should not normally be used because it will
* fail to correctly associate any comments with the token being passed in.
*
* @param token the token to be appended
*/
protected void appendToken(Token token) {
tail = tail.setNext(token);
}
protected int bigSwitch(int next) {
beginToken();
if (next == '\r') {
next = reader.advance();
if (next == '\n') {
next = reader.advance();
}
recordStartOfLine();
return next;
} else if (next == '\n') {
next = reader.advance();
recordStartOfLine();
return next;
} else if (next == '\t' || next == ' ') {
return reader.advance();
}
if (next == 'r') {
int peek = reader.peek();
if (peek == '"' || peek == '\'') {
int start = reader.getOffset();
return tokenizeString(reader.advance(), start, true);
}
}
if ('a' <= next && next <= 'z') {
return tokenizeKeywordOrIdentifier(next, true);
}
if (('A' <= next && next <= 'Z') || next == '_' || next == '$') {
return tokenizeIdentifier(next, reader.getOffset(), true);
}
if (next == '<') {
return tokenizeLessThan(next);
}
if (next == '>') {
return tokenizeGreaterThan(next);
}
if (next == '=') {
return tokenizeEquals(next);
}
if (next == '!') {
return tokenizeExclamation(next);
}
if (next == '+') {
return tokenizePlus(next);
}
if (next == '-') {
return tokenizeMinus(next);
}
if (next == '*') {
return tokenizeMultiply(next);
}
if (next == '%') {
return tokenizePercent(next);
}
if (next == '&') {
return tokenizeAmpersand(next);
}
if (next == '|') {
return tokenizeBar(next);
}
if (next == '^') {
return tokenizeCaret(next);
}
if (next == '[') {
return tokenizeOpenSquareBracket(next);
}
if (next == '~') {
return tokenizeTilde(next);
}
if (next == '\\') {
appendTokenOfType(TokenType.BACKSLASH);
return reader.advance();
}
if (next == '#') {
return tokenizeTag(next);
}
if (next == '(') {
appendBeginToken(TokenType.OPEN_PAREN);
return reader.advance();
}
if (next == ')') {
appendEndToken(TokenType.CLOSE_PAREN, TokenType.OPEN_PAREN);
return reader.advance();
}
if (next == ',') {
appendTokenOfType(TokenType.COMMA);
return reader.advance();
}
if (next == ':') {
appendTokenOfType(TokenType.COLON);
return reader.advance();
}
if (next == ';') {
appendTokenOfType(TokenType.SEMICOLON);
return reader.advance();
}
if (next == '?') {
appendTokenOfType(TokenType.QUESTION);
return reader.advance();
}
if (next == ']') {
appendEndToken(TokenType.CLOSE_SQUARE_BRACKET, TokenType.OPEN_SQUARE_BRACKET);
return reader.advance();
}
if (next == '`') {
appendTokenOfType(TokenType.BACKPING);
return reader.advance();
}
if (next == '{') {
appendBeginToken(TokenType.OPEN_CURLY_BRACKET);
return reader.advance();
}
if (next == '}') {
appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRACKET);
return reader.advance();
}
if (next == '/') {
return tokenizeSlashOrComment(next);
}
if (next == '@') {
appendTokenOfType(TokenType.AT);
return reader.advance();
}
if (next == '"' || next == '\'') {
return tokenizeString(next, reader.getOffset(), false);
}
if (next == '.') {
return tokenizeDotOrNumber(next);
}
if (next == '0') {
return tokenizeHexOrNumber(next);
}
if ('1' <= next && next <= '9') {
return tokenizeNumber(next);
}
if (next == -1) {
return -1;
}
reportError(ScannerErrorCode.ILLEGAL_CHARACTER, Integer.valueOf(next));
return reader.advance();
}
/**
* Return the first token in the token stream that was scanned.
*
* @return the first token in the token stream that was scanned
*/
protected Token getFirstToken() {
return tokens.getNext();
}
/**
* Return the last token that was scanned.
*
* @return the last token that was scanned
*/
protected Token getTail() {
return tail;
}
/**
* Record the fact that we are at the beginning of a new line in the source.
*/
protected void recordStartOfLine() {
lineStarts.add(reader.getOffset());
}
private void appendBeginToken(TokenType type) {
BeginToken token;
if (firstComment == null) {
token = new BeginToken(type, tokenStart);
} else {
token = new BeginTokenWithComment(type, tokenStart, firstComment);
firstComment = null;
lastComment = null;
}
tail = tail.setNext(token);
groupingStack.add(token);
stackEnd++;
}
private void appendCommentToken(TokenType type, String value) {
// Ignore comment tokens if client specified that it doesn't need them.
if (!preserveComments) {
return;
}
// OK, remember comment tokens.
if (firstComment == null) {
firstComment = new StringToken(type, value, tokenStart);
lastComment = firstComment;
} else {
lastComment = lastComment.setNext(new StringToken(type, value, tokenStart));
}
}
private void appendEndToken(TokenType type, TokenType beginType) {
Token token;
if (firstComment == null) {
token = new Token(type, tokenStart);
} else {
token = new TokenWithComment(type, tokenStart, firstComment);
firstComment = null;
lastComment = null;
}
tail = tail.setNext(token);
if (stackEnd >= 0) {
BeginToken begin = groupingStack.get(stackEnd);
if (begin.getType() == beginType) {
begin.setEndToken(token);
groupingStack.remove(stackEnd--);
}
}
}
private void appendEofToken() {
Token eofToken;
if (firstComment == null) {
eofToken = new Token(TokenType.EOF, reader.getOffset() + 1);
} else {
eofToken = new TokenWithComment(TokenType.EOF, reader.getOffset() + 1, firstComment);
firstComment = null;
lastComment = null;
}
// The EOF token points to itself so that there is always infinite look-ahead.
eofToken.setNext(eofToken);
tail = tail.setNext(eofToken);
if (stackEnd >= 0) {
hasUnmatchedGroups = true;
// TODO(brianwilkerson) Fix the ungrouped tokens?
}
}
private void appendKeywordToken(Keyword keyword) {
if (firstComment == null) {
tail = tail.setNext(new KeywordToken(keyword, tokenStart));
} else {
tail = tail.setNext(new KeywordTokenWithComment(keyword, tokenStart, firstComment));
firstComment = null;
lastComment = null;
}
}
private void appendStringToken(TokenType type, String value) {
if (firstComment == null) {
tail = tail.setNext(new StringToken(type, value, tokenStart));
} else {
tail = tail.setNext(new StringTokenWithComment(type, value, tokenStart, firstComment));
firstComment = null;
lastComment = null;
}
}
private void appendStringTokenWithOffset(TokenType type, String value, int offset) {
if (firstComment == null) {
tail = tail.setNext(new StringToken(type, value, tokenStart + offset));
} else {
tail = tail.setNext(new StringTokenWithComment(type, value, tokenStart + offset, firstComment));
firstComment = null;
lastComment = null;
}
}
private void appendTokenOfType(TokenType type) {
if (firstComment == null) {
tail = tail.setNext(new Token(type, tokenStart));
} else {
tail = tail.setNext(new TokenWithComment(type, tokenStart, firstComment));
firstComment = null;
lastComment = null;
}
}
private void appendTokenOfTypeWithOffset(TokenType type, int offset) {
if (firstComment == null) {
tail = tail.setNext(new Token(type, offset));
} else {
tail = tail.setNext(new TokenWithComment(type, offset, firstComment));
firstComment = null;
lastComment = null;
}
}
private void beginToken() {
tokenStart = reader.getOffset();
}
/**
* Return the beginning token corresponding to a closing brace that was found while scanning
* inside a string interpolation expression. Tokens that cannot be matched with the closing brace
* will be dropped from the stack.
*
* @return the token to be paired with the closing brace
*/
private BeginToken findTokenMatchingClosingBraceInInterpolationExpression() {
while (stackEnd >= 0) {
BeginToken begin = groupingStack.get(stackEnd);
if (begin.getType() == TokenType.OPEN_CURLY_BRACKET
|| begin.getType() == TokenType.STRING_INTERPOLATION_EXPRESSION) {
return begin;
}
hasUnmatchedGroups = true;
groupingStack.remove(stackEnd--);
}
//
// We should never get to this point because we wouldn't be inside a string interpolation
// expression unless we had previously found the start of the expression.
//
return null;
}
/**
* Return the source being scanned.
*
* @return the source being scanned
*/
private Source getSource() {
return source;
}
/**
* Report an error at the current offset.
*
* @param errorCode the error code indicating the nature of the error
* @param arguments any arguments needed to complete the error message
*/
private void reportError(ScannerErrorCode errorCode, Object... arguments) {
errorListener.onError(new AnalysisError(
getSource(),
reader.getOffset(),
1,
errorCode,
arguments));
}
private int select(char choice, TokenType yesType, TokenType noType) {
int next = reader.advance();
if (next == choice) {
appendTokenOfType(yesType);
return reader.advance();
} else {
appendTokenOfType(noType);
return next;
}
}
private int selectWithOffset(char choice, TokenType yesType, TokenType noType, int offset) {
int next = reader.advance();
if (next == choice) {
appendTokenOfTypeWithOffset(yesType, offset);
return reader.advance();
} else {
appendTokenOfTypeWithOffset(noType, offset);
return next;
}
}
private int tokenizeAmpersand(int next) {
// && &= &
next = reader.advance();
if (next == '&') {
appendTokenOfType(TokenType.AMPERSAND_AMPERSAND);
return reader.advance();
} else if (next == '=') {
appendTokenOfType(TokenType.AMPERSAND_EQ);
return reader.advance();
} else {
appendTokenOfType(TokenType.AMPERSAND);
return next;
}
}
private int tokenizeBar(int next) {
// | || |=
next = reader.advance();
if (next == '|') {
appendTokenOfType(TokenType.BAR_BAR);
return reader.advance();
} else if (next == '=') {
appendTokenOfType(TokenType.BAR_EQ);
return reader.advance();
} else {
appendTokenOfType(TokenType.BAR);
return next;
}
}
private int tokenizeCaret(int next) {
// ^ ^=
return select('=', TokenType.CARET_EQ, TokenType.CARET);
}
private int tokenizeDotOrNumber(int next) {
int start = reader.getOffset();
next = reader.advance();
if (('0' <= next && next <= '9')) {
return tokenizeFractionPart(next, start);
} else if ('.' == next) {
return select('.', TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERIOD);
} else {
appendTokenOfType(TokenType.PERIOD);
return next;
}
}
private int tokenizeEquals(int next) {
// = == =>
next = reader.advance();
if (next == '=') {
appendTokenOfType(TokenType.EQ_EQ);
return reader.advance();
} else if (next == '>') {
appendTokenOfType(TokenType.FUNCTION);
return reader.advance();
}
appendTokenOfType(TokenType.EQ);
return next;
}
private int tokenizeExclamation(int next) {
// ! !=
next = reader.advance();
if (next == '=') {
appendTokenOfType(TokenType.BANG_EQ);
return reader.advance();
}
appendTokenOfType(TokenType.BANG);
return next;
}
private int tokenizeExponent(int next) {
if (next == '+' || next == '-') {
next = reader.advance();
}
boolean hasDigits = false;
while (true) {
if ('0' <= next && next <= '9') {
hasDigits = true;
} else {
if (!hasDigits) {
reportError(ScannerErrorCode.MISSING_DIGIT);
}
return next;
}
next = reader.advance();
}
}
private int tokenizeFractionPart(int next, int start) {
boolean done = false;
boolean hasDigit = false;
LOOP : while (!done) {
if ('0' <= next && next <= '9') {
hasDigit = true;
} else if ('e' == next || 'E' == next) {
hasDigit = true;
next = tokenizeExponent(reader.advance());
done = true;
continue LOOP;
} else {
done = true;
continue LOOP;
}
next = reader.advance();
}
if (!hasDigit) {
appendStringToken(TokenType.INT, reader.getString(start, -2));
if ('.' == next) {
return selectWithOffset(
'.',
TokenType.PERIOD_PERIOD_PERIOD,
TokenType.PERIOD_PERIOD,
reader.getOffset() - 1);
}
appendTokenOfTypeWithOffset(TokenType.PERIOD, reader.getOffset() - 1);
return bigSwitch(next);
}
appendStringToken(TokenType.DOUBLE, reader.getString(start, next < 0 ? 0 : -1));
return next;
}
private int tokenizeGreaterThan(int next) {
// > >= >> >>=
next = reader.advance();
if ('=' == next) {
appendTokenOfType(TokenType.GT_EQ);
return reader.advance();
} else if ('>' == next) {
next = reader.advance();
if ('=' == next) {
appendTokenOfType(TokenType.GT_GT_EQ);
return reader.advance();
} else {
appendTokenOfType(TokenType.GT_GT);
return next;
}
} else {
appendTokenOfType(TokenType.GT);
return next;
}
}
private int tokenizeHex(int next) {
int start = reader.getOffset() - 1;
boolean hasDigits = false;
while (true) {
next = reader.advance();
if (('0' <= next && next <= '9') || ('A' <= next && next <= 'F')
|| ('a' <= next && next <= 'f')) {
hasDigits = true;
} else {
if (!hasDigits) {
reportError(ScannerErrorCode.MISSING_HEX_DIGIT);
}
appendStringToken(TokenType.HEXADECIMAL, reader.getString(start, next < 0 ? 0 : -1));
return next;
}
}
}
private int tokenizeHexOrNumber(int next) {
int x = reader.peek();
if (x == 'x' || x == 'X') {
reader.advance();
return tokenizeHex(x);
}
return tokenizeNumber(next);
}
private int tokenizeIdentifier(int next, int start, boolean allowDollar) {
while (('a' <= next && next <= 'z') || ('A' <= next && next <= 'Z')
|| ('0' <= next && next <= '9') || next == '_' || (next == '$' && allowDollar)) {
next = reader.advance();
}
appendStringToken(TokenType.IDENTIFIER, reader.getString(start, next < 0 ? 0 : -1));
return next;
}
private int tokenizeInterpolatedExpression(int next, int start) {
appendBeginToken(TokenType.STRING_INTERPOLATION_EXPRESSION);
next = reader.advance();
while (next != -1) {
if (next == '}') {
BeginToken begin = findTokenMatchingClosingBraceInInterpolationExpression();
if (begin == null) {
beginToken();
appendTokenOfType(TokenType.CLOSE_CURLY_BRACKET);
next = reader.advance();
beginToken();
return next;
} else if (begin.getType() == TokenType.OPEN_CURLY_BRACKET) {
beginToken();
appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRACKET);
next = reader.advance();
beginToken();
} else if (begin.getType() == TokenType.STRING_INTERPOLATION_EXPRESSION) {
beginToken();
appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.STRING_INTERPOLATION_EXPRESSION);
next = reader.advance();
beginToken();
return next;
}
} else {
next = bigSwitch(next);
}
}
return next;
}
private int tokenizeInterpolatedIdentifier(int next, int start) {
appendStringTokenWithOffset(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 0);
if ((('A' <= next && next <= 'Z') || ('a' <= next && next <= 'z') || next == '_')) {
beginToken();
next = tokenizeKeywordOrIdentifier(next, false);
}
beginToken();
return next;
}
private int tokenizeKeywordOrIdentifier(int next, boolean allowDollar) {
KeywordState state = KeywordState.KEYWORD_STATE;
int start = reader.getOffset();
while (state != null && 'a' <= next && next <= 'z') {
state = state.next((char) next);
next = reader.advance();
}
if (state == null || state.keyword() == null) {
return tokenizeIdentifier(next, start, allowDollar);
}
if (('A' <= next && next <= 'Z') || ('0' <= next && next <= '9') || next == '_' || next == '$') {
return tokenizeIdentifier(next, start, allowDollar);
} else if (next < 128) {
appendKeywordToken(state.keyword());
return next;
} else {
return tokenizeIdentifier(next, start, allowDollar);
}
}
private int tokenizeLessThan(int next) {
// < <= << <<=
next = reader.advance();
if ('=' == next) {
appendTokenOfType(TokenType.LT_EQ);
return reader.advance();
} else if ('<' == next) {
return select('=', TokenType.LT_LT_EQ, TokenType.LT_LT);
} else {
appendTokenOfType(TokenType.LT);
return next;
}
}
private int tokenizeMinus(int next) {
// - -- -=
next = reader.advance();
if (next == '-') {
appendTokenOfType(TokenType.MINUS_MINUS);
return reader.advance();
} else if (next == '=') {
appendTokenOfType(TokenType.MINUS_EQ);
return reader.advance();
} else {
appendTokenOfType(TokenType.MINUS);
return next;
}
}
private int tokenizeMultiLineComment(int next) {
int nesting = 1;
next = reader.advance();
while (true) {
if (-1 == next) {
reportError(ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT);
appendCommentToken(TokenType.MULTI_LINE_COMMENT, reader.getString(tokenStart, 0));
return next;
} else if ('*' == next) {
next = reader.advance();
if ('/' == next) {
--nesting;
if (0 == nesting) {
appendCommentToken(TokenType.MULTI_LINE_COMMENT, reader.getString(tokenStart, 0));
return reader.advance();
} else {
next = reader.advance();
}
}
} else if ('/' == next) {
next = reader.advance();
if ('*' == next) {
next = reader.advance();
++nesting;
}
} else if (next == '\r') {
next = reader.advance();
if (next == '\n') {
next = reader.advance();
}
recordStartOfLine();
} else if (next == '\n') {
recordStartOfLine();
next = reader.advance();
} else {
next = reader.advance();
}
}
}
private int tokenizeMultiLineRawString(int quoteChar, int start) {
int next = reader.advance();
outer : while (next != -1) {
while (next != quoteChar) {
next = reader.advance();
if (next == -1) {
break outer;
} else if (next == '\r') {
next = reader.advance();
if (next == '\n') {
next = reader.advance();
}
recordStartOfLine();
} else if (next == '\n') {
recordStartOfLine();
next = reader.advance();
}
}
next = reader.advance();
if (next == quoteChar) {
next = reader.advance();
if (next == quoteChar) {
appendStringToken(TokenType.STRING, reader.getString(start, 0));
return reader.advance();
}
}
}
reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL);
appendStringToken(TokenType.STRING, reader.getString(start, 0));
return reader.advance();
}
private int tokenizeMultiLineString(int quoteChar, int start, boolean raw) {
if (raw) {
return tokenizeMultiLineRawString(quoteChar, start);
}
int next = reader.advance();
while (next != -1) {
if (next == '$') {
appendStringToken(TokenType.STRING, reader.getString(start, -1));
next = tokenizeStringInterpolation(start);
beginToken();
start = reader.getOffset();
continue;
}
if (next == quoteChar) {
next = reader.advance();
if (next == quoteChar) {
next = reader.advance();
if (next == quoteChar) {
appendStringToken(TokenType.STRING, reader.getString(start, 0));
return reader.advance();
}
}
continue;
}
if (next == '\\') {
next = reader.advance();
if (next == -1) {
break;
}
if (next == '\r') {
next = reader.advance();
if (next == '\n') {
next = reader.advance();
}
recordStartOfLine();
} else if (next == '\n') {
recordStartOfLine();
next = reader.advance();
} else {
next = reader.advance();
}
} else if (next == '\r') {
next = reader.advance();
if (next == '\n') {
next = reader.advance();
}
recordStartOfLine();
} else if (next == '\n') {
recordStartOfLine();
next = reader.advance();
} else {
next = reader.advance();
}
}
reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL);
if (start == reader.getOffset()) {
appendStringTokenWithOffset(TokenType.STRING, "", 1);
} else {
appendStringToken(TokenType.STRING, reader.getString(start, 0));
}
return reader.advance();
}
private int tokenizeMultiply(int next) {
// * *=
return select('=', TokenType.STAR_EQ, TokenType.STAR);
}
private int tokenizeNumber(int next) {
int start = reader.getOffset();
while (true) {
next = reader.advance();
if ('0' <= next && next <= '9') {
continue;
} else if (next == '.') {
return tokenizeFractionPart(reader.advance(), start);
} else if (next == 'e' || next == 'E') {
return tokenizeFractionPart(next, start);
} else {
appendStringToken(TokenType.INT, reader.getString(start, next < 0 ? 0 : -1));
return next;
}
}
}
private int tokenizeOpenSquareBracket(int next) {
// [ [] []=
next = reader.advance();
if (next == ']') {
return select('=', TokenType.INDEX_EQ, TokenType.INDEX);
} else {
appendBeginToken(TokenType.OPEN_SQUARE_BRACKET);
return next;
}
}
private int tokenizePercent(int next) {
// % %=
return select('=', TokenType.PERCENT_EQ, TokenType.PERCENT);
}
private int tokenizePlus(int next) {
// + ++ +=
next = reader.advance();
if ('+' == next) {
appendTokenOfType(TokenType.PLUS_PLUS);
return reader.advance();
} else if ('=' == next) {
appendTokenOfType(TokenType.PLUS_EQ);
return reader.advance();
} else {
appendTokenOfType(TokenType.PLUS);
return next;
}
}
private int tokenizeSingleLineComment(int next) {
while (true) {
next = reader.advance();
if (-1 == next) {
appendCommentToken(TokenType.SINGLE_LINE_COMMENT, reader.getString(tokenStart, 0));
return next;
} else if ('\n' == next || '\r' == next) {
appendCommentToken(TokenType.SINGLE_LINE_COMMENT, reader.getString(tokenStart, -1));
return next;
}
}
}
private int tokenizeSingleLineRawString(int next, int quoteChar, int start) {
next = reader.advance();
while (next != -1) {
if (next == quoteChar) {
appendStringToken(TokenType.STRING, reader.getString(start, 0));
return reader.advance();
} else if (next == '\r' || next == '\n') {
reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL);
appendStringToken(TokenType.STRING, reader.getString(start, -1));
return reader.advance();
}
next = reader.advance();
}
reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL);
appendStringToken(TokenType.STRING, reader.getString(start, 0));
return reader.advance();
}
private int tokenizeSingleLineString(int next, int quoteChar, int start) {
while (next != quoteChar) {
if (next == '\\') {
next = reader.advance();
} else if (next == '$') {
appendStringToken(TokenType.STRING, reader.getString(start, -1));
next = tokenizeStringInterpolation(start);
beginToken();
start = reader.getOffset();
continue;
}
if (next <= '\r' && (next == '\n' || next == '\r' || next == -1)) {
reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL);
if (start == reader.getOffset()) {
appendStringTokenWithOffset(TokenType.STRING, "", 1);
} else if (next == -1) {
appendStringToken(TokenType.STRING, reader.getString(start, 0));
} else {
appendStringToken(TokenType.STRING, reader.getString(start, -1));
}
return reader.advance();
}
next = reader.advance();
}
appendStringToken(TokenType.STRING, reader.getString(start, 0));
return reader.advance();
}
private int tokenizeSlashOrComment(int next) {
next = reader.advance();
if ('*' == next) {
return tokenizeMultiLineComment(next);
} else if ('/' == next) {
return tokenizeSingleLineComment(next);
} else if ('=' == next) {
appendTokenOfType(TokenType.SLASH_EQ);
return reader.advance();
} else {
appendTokenOfType(TokenType.SLASH);
return next;
}
}
private int tokenizeString(int next, int start, boolean raw) {
int quoteChar = next;
next = reader.advance();
if (quoteChar == next) {
next = reader.advance();
if (quoteChar == next) {
// Multiline string.
return tokenizeMultiLineString(quoteChar, start, raw);
} else {
// Empty string.
appendStringToken(TokenType.STRING, reader.getString(start, -1));
return next;
}
}
if (raw) {
return tokenizeSingleLineRawString(next, quoteChar, start);
} else {
return tokenizeSingleLineString(next, quoteChar, start);
}
}
private int tokenizeStringInterpolation(int start) {
beginToken();
int next = reader.advance();
if (next == '{') {
return tokenizeInterpolatedExpression(next, start);
} else {
return tokenizeInterpolatedIdentifier(next, start);
}
}
private int tokenizeTag(int next) {
// # or #!.*[\n\r]
if (reader.getOffset() == 0) {
if (reader.peek() == '!') {
do {
next = reader.advance();
} while (next != '\n' && next != '\r' && next > 0);
appendStringToken(TokenType.SCRIPT_TAG, reader.getString(tokenStart, 0));
return next;
}
}
appendTokenOfType(TokenType.HASH);
return reader.advance();
}
private int tokenizeTilde(int next) {
// ~ ~/ ~/=
next = reader.advance();
if (next == '/') {
return select('=', TokenType.TILDE_SLASH_EQ, TokenType.TILDE_SLASH);
} else {
appendTokenOfType(TokenType.TILDE);
return next;
}
}
}