/*******************************************************************************
* Copyright (c) 2009, 2015, 2016, 2017 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* IBM Corporation - initial API and implementation
* Zend Technologies
*******************************************************************************/
package org.eclipse.php.internal.core.documentModel.parser.regions;
import java.util.*;
import org.eclipse.dltk.annotations.Nullable;
import org.eclipse.jface.text.BadLocationException;
import org.eclipse.php.internal.core.documentModel.parser.AbstractPHPLexer;
import org.eclipse.php.internal.core.documentModel.parser.Scanner.LexerState;
import org.eclipse.php.internal.core.documentModel.partitioner.PHPPartitionTypes;
import org.eclipse.wst.sse.core.internal.parser.ContextRegion;
import org.eclipse.wst.sse.core.internal.provisional.text.ITextRegion;
/**
* Description: Holds the tokens extracted from the script
*
* @author Roy, 2007
*/
public class PHPTokenContainer implements Cloneable {
// holds PHP tokens
protected final LinkedList<ContextRegion> phpTokens = new LinkedList<ContextRegion>(); // of
// ITextRegion
// holds the location and state, where the lexical analyzer state was
// changed
protected final LinkedList<LexerStateChange> lexerStateChanges = new LinkedList<LexerStateChange>(); // of
// LexerStateChanged
// holds the iterator for the php tokens linked list
// this iterator follows the localization principle
// i.e. the user usually works in the same area of the document
protected ListIterator<ContextRegion> tokensIterator = null;
// https://bugs.eclipse.org/bugs/show_bug.cgi?id=464489
// workaround for bug 464489
@Override
public Object clone() {
PHPTokenContainer clone = new PHPTokenContainer();
clone.getModelForCreation();
clone.reset();
clone.phpTokens.addAll(phpTokens);
clone.lexerStateChanges.addAll(lexerStateChanges);
clone.releaseModelFromCreation();
return clone;
}
/**
* find token for a given location
*
* @param offset
* @return token (will never be null)
* @throws BadLocationException
* - if the offset is out of bound
*/
public synchronized ITextRegion getToken(int offset) throws BadLocationException {
assert tokensIterator != null;
if (phpTokens.isEmpty()) {
throw new BadLocationException("offset " + offset + " cannot be contained in an empty region"); //$NON-NLS-1$ //$NON-NLS-2$
}
// we have at least one region...
checkBadLocation(offset);
// smart searching
ITextRegion result = tokensIterator.hasNext() ? tokensIterator.next() : tokensIterator.previous();
assert result != null && result.getLength() > 0;
if (isInside(result, offset)) {
return result;
}
if (offset >= result.getEnd()) { // if the offset is
// beyond - go fetch
// from next
while (tokensIterator.hasNext() && !isInside(result, offset)) {
result = tokensIterator.next();
assert result != null && result.getLength() > 0;
}
} else { // else go fetch from previous
while (tokensIterator.hasPrevious() && !isInside(result, offset)) {
result = tokensIterator.previous();
assert result != null && result.getLength() > 0;
}
// moves the iterator to the next one
if (tokensIterator.hasNext()) {
tokensIterator.next();
}
}
assert isInside(result, offset) || offset == getLastToken().getEnd();
return result;
}
public synchronized ITextRegion[] getTokens(final int offset, final int length) throws BadLocationException {
if (length < 0) {
throw new BadLocationException("length " + length + " cannot be < 0"); //$NON-NLS-1$ //$NON-NLS-2$
}
List<ITextRegion> result = new ArrayList<ITextRegion>(); // list of
// ITextRegion
ITextRegion token = phpTokens.isEmpty() ? null : getToken(offset);
if (token != null) {
result.add(token);
}
while (tokensIterator.hasNext() && token != null && token.getEnd() < offset + length) {
token = tokensIterator.next();
assert token != null;
result.add(token);
}
return result.toArray(new ITextRegion[result.size()]);
}
private final boolean isInside(ITextRegion region, int offset) {
return region != null && region.getStart() <= offset && offset < region.getEnd();
}
/**
* @param offset
* @return the lexer state at the given offset
*/
public synchronized @Nullable LexerState getState(int offset) {
Iterator<LexerStateChange> iter = lexerStateChanges.iterator();
assert iter.hasNext();
LexerStateChange element = iter.next();
LexerState lastState = null;
while (offset >= element.getOffset()) {
lastState = element.state;
if (!iter.hasNext()) {
return lastState;
}
element = iter.next();
}
return lastState;
}
/**
* @param offset
* @return the partition type of the given offset (will never be null)
* @throws BadLocationException
*/
public synchronized String getPartitionType(int offset) throws BadLocationException {
ITextRegion token = getToken(offset);
assert token != null;
// while (PHPRegionTypes.PHPDOC_TODO.equals(token.getType()) &&
// token.getStart() - 1 >= 0) {
// token = getToken(token.getStart() - 1);
// assert token != null;
// }
final String type = token.getType();
final String partitionType = PHPPartitionTypes.getPartitionType(type);
assert partitionType != null;
return partitionType;
}
/**
* Updates the container with the new states
*
* @param newContainer
* @param fromOffset
* @param toOffset
* @param size
*/
public synchronized void updateStateChanges(PHPTokenContainer newContainer, int fromOffset, int toOffset) {
if (newContainer.lexerStateChanges.size() < 2) {
return;
}
// remove
final ListIterator<LexerStateChange> oldIterator = removeOldChanges(fromOffset, toOffset);
// add
final Iterator<LexerStateChange> newIterator = newContainer.lexerStateChanges.iterator();
newIterator.next(); // ignore the first state change (it is identical to
// the original one)
// goto the previous before adding
setIterator(oldIterator, fromOffset, toOffset);
while (newIterator.hasNext()) {
oldIterator.add(newIterator.next());
}
}
private void setIterator(ListIterator<LexerStateChange> oldIterator, int fromOffset, int toOffset) {
if (oldIterator.nextIndex() != 1) {
oldIterator.previous();
} else {
return;
}
LexerStateChange next = oldIterator.next();
int offset = next.getOffset();
if (offset > fromOffset) {
oldIterator.previous();
}
}
public synchronized ListIterator<ContextRegion> removeTokensSubList(ITextRegion tokenStart, ITextRegion tokenEnd) {
assert tokenStart != null;
// go to the start region
ITextRegion region = null;
;
try {
region = getToken(tokenStart.getStart());
} catch (BadLocationException e) {
assert false;
}
assert region == tokenStart;
tokensIterator.remove();
// if it the start and the end are equal - remove and exit
if (tokenStart != tokenEnd) {
// remove to the end
do {
region = tokensIterator.next();
tokensIterator.remove();
} while (tokensIterator.hasNext() && region != tokenEnd);
}
return tokensIterator;
}
/**
* One must call getModelForWrite() in order to construct the list of php
* tokens
*/
public synchronized void getModelForCreation() {
tokensIterator = null;
}
/**
* One must call releaseModelForWrite() after constructing the list of php
* tokens
*/
public synchronized void releaseModelFromCreation() {
tokensIterator = phpTokens.listIterator();
}
/**
* Returns an read-only iterator to the php tokens, calling next() returns
* the first token in the wanted offset
*
* @param offset
* @param length
* @return
* @throws BadLocationException
*/
public synchronized ListIterator<ContextRegion> getPHPTokensIterator(final int offset) throws BadLocationException {
// fast results for empty lists
if (phpTokens.isEmpty()) {
return tokensIterator;
}
checkBadLocation(offset);
// set the token iterator to the right place
getToken(offset);
return tokensIterator;
}
/**
* @return the whole tokens as an array
*/
public synchronized ITextRegion[] getPHPTokens() {
return phpTokens.toArray(new ITextRegion[phpTokens.size()]);
}
/**
* Clears the containers
*/
public synchronized void reset() {
this.phpTokens.clear();
this.lexerStateChanges.clear();
}
/**
* @return true for empty container
*/
public synchronized boolean isEmpty() {
return this.phpTokens.isEmpty();
}
/**
* Push region to the end of the tokens list
*
* @param region
* @param lexerState
*/
public synchronized void addLast(String yylex, int start, int yylengthLength, int yylength, LexerState lexerState) {
assert (phpTokens.size() == 0 || getLastToken().getEnd() == start) && tokensIterator == null;
if (phpTokens.size() > 0) {
ContextRegion lastContextRegion = (ContextRegion) phpTokens.get(phpTokens.size() - 1);
if (deprecatedKeywordAfter(lastContextRegion.getType())) {
if (isKeyword(yylex)) {
yylex = PHPRegionTypes.PHP_LABEL;
}
}
}
// if state was change - we add a new token and add state
if (lexerStateChanges.size() == 0 || !getLastChange().state.equals(lexerState)) {
int textLength = yylengthLength;
if (yylex == AbstractPHPLexer.WHITESPACE) {
textLength = 0;
} else if (yylex == AbstractPHPLexer.PHP_CURLY_OPEN || yylex == AbstractPHPLexer.PHP_CURLY_CLOSE) {
textLength = 1;
}
final ContextRegion contextRegion = new ContextRegion(yylex, start, textLength, yylength);
phpTokens.addLast(contextRegion);
lexerStateChanges.addLast(new LexerStateChange(lexerState, contextRegion));
return;
}
assert phpTokens.size() > 0;
// if we can only adjust the previous token size
if (yylex == AbstractPHPLexer.WHITESPACE) {
final ITextRegion last = phpTokens.getLast();
last.adjustLength(yylength);
} else { // else - add as a new token
int textLength = yylengthLength;
if (yylex == AbstractPHPLexer.PHP_CURLY_OPEN || yylex == AbstractPHPLexer.PHP_CURLY_CLOSE) {
textLength = 1;
}
final ContextRegion contextRegion = new ContextRegion(yylex, start, textLength, yylength);
phpTokens.addLast(contextRegion);
}
}
/**
* if the keyword could be use as identifier
*
* @param yylex
* @return if yylex is one of the keywords that could be use as identifier
*/
public static boolean isKeyword(String yylex) {
if (PHPRegionTypes.PHP_FROM.equals(yylex)) {
return true;
}
return false;
}
/**
* if the keyword should be a normal identifier after special type
*
* @param yylex
* @return if the keyword should be a normal identifier after yylex
*/
public static boolean deprecatedKeywordAfter(String yylex) {
if (PHPRegionTypes.PHP_FUNCTION.equals(yylex) || PHPRegionTypes.PHP_CONST.equals(yylex)) {
return true;
}
return false;
}
/**
* Adjust the length of the last token for whitespace tokens
*
* @param yylex
* @param start
* @param yylengthLength
* @param yylength
* @param lexerState
*/
public synchronized void adjustWhitespace(String yylex, int start, int yylengthLength, int yylength,
Object lexerState) {
assert (phpTokens.size() == 0 || getLastToken().getEnd() == start) && tokensIterator == null;
// if state was change - we add a new token and add state
if (lexerStateChanges.size() != 0 && getLastChange().state.equals(lexerState)) {
final ITextRegion last = phpTokens.getLast();
last.adjustLength(yylength);
}
}
/**
* This node represent a change in the lexer state during lexical analysis
*/
protected static final class LexerStateChange {
public final LexerState state;
public final ITextRegion firstRegion;
public LexerStateChange(final LexerState state, final ITextRegion firstRegion) {
assert firstRegion != null && state != null;
this.state = state;
this.firstRegion = firstRegion;
}
public final int getOffset() {
return firstRegion.getStart();
}
public int hashCode() {
return 31 + ((state == null) ? 0 : state.hashCode());
}
public boolean equals(final Object obj) {
assert state != null && obj.getClass() == LexerState.class;
if (this.state == obj)
return true;
return state.equals((LexerState) obj);
}
public final String toString() {
return "[" + getOffset() + "] - " + this.state.getTopState(); //$NON-NLS-1$ //$NON-NLS-2$
}
}
/**
* check for out of bound
*
* @param offset
* @throws BadLocationException
*/
protected synchronized final void checkBadLocation(int offset) throws BadLocationException {
ITextRegion lastRegion = getLastToken();
if (offset < 0 || lastRegion.getEnd() < offset) {
throw new BadLocationException("offset " + offset + " is out of [0, " + lastRegion.getEnd() + "]"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
}
}
protected synchronized final ITextRegion getLastToken() {
return phpTokens.getLast();
}
protected synchronized LexerStateChange getLastChange() {
return lexerStateChanges.getLast();
}
protected synchronized final ListIterator<LexerStateChange> removeOldChanges(int fromOffset, int toOffset) {
final ListIterator<LexerStateChange> iterator = (ListIterator<LexerStateChange>) lexerStateChanges.iterator();
LexerStateChange element = iterator.next();
while (element.getOffset() <= toOffset) {
if (element.getOffset() > fromOffset && element.getOffset() <= toOffset) {
iterator.remove();
}
if (!iterator.hasNext()) {
return iterator;
}
element = iterator.next();
}
return iterator;
}
}