TokenManager.java

/*******************************************************************************
 * Copyright (c) 2014, 2016 Mateusz Matela and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     Mateusz Matela <mateusz.matela@gmail.com> - [formatter] Formatter does not format Java code correctly, especially when max line width is set - https://bugs.eclipse.org/303519
 *******************************************************************************/
package org.eclipse.jdt.internal.formatter;

import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_BLOCK;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_JAVADOC;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameLBRACE;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameNotAToken;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameStringLiteral;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameWHITESPACE;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.eclipse.jdt.core.dom.ASTNode;
import org.eclipse.jdt.core.dom.Block;
import org.eclipse.jdt.core.dom.IfStatement;
import org.eclipse.jdt.core.dom.ReturnStatement;
import org.eclipse.jdt.core.dom.ThrowStatement;
import org.eclipse.jdt.internal.formatter.Token.WrapMode;
import org.eclipse.jdt.internal.formatter.linewrap.CommentWrapExecutor;

/**
 * A helper class that can be used to easily access source code and find tokens on any position.
 * It also has some other methods that are useful on multiple stages of formatting.
 */
public class TokenManager implements Iterable<Token> {

	private static final Pattern COMMENT_LINE_ANNOTATION_PATTERN = Pattern.compile("^(\\s*\\*?\\s*)(@)"); //$NON-NLS-1$

	private final List<Token> tokens;
	private final String source;
	private final int tabSize;
	private final int tabChar;
	private final boolean wrapWithSpaces;

	final CommentWrapExecutor commentWrapper;

	private HashMap<Integer, Integer> tokenIndexToNLSAlign;
	private List<Token[]> formatOffTagPairs;
	private int headerEndIndex = 0;

	public TokenManager(List<Token> tokens, String source, DefaultCodeFormatterOptions options) {
		this.tokens = tokens;
		this.source = source;
		this.tabSize = options.tab_size;
		this.tabChar = options.tab_char;
		this.wrapWithSpaces = options.use_tabs_only_for_leading_indentations;
		this.commentWrapper = new CommentWrapExecutor(this, options);
	}

	public TokenManager(List<Token> tokens, TokenManager parent) {
		this.tokens = tokens;
		this.source = parent.source;
		this.tabSize = parent.tabSize;
		this.tabChar = parent.tabChar;
		this.wrapWithSpaces = parent.wrapWithSpaces;
		this.commentWrapper = parent.commentWrapper;
	}

	public Token get(int index) {
		return this.tokens.get(index);
	}

	/**
	 * @return total number of tokens
	 */
	public int size() {
		return this.tokens.size();
	}

	/**
	 * Removes the token at given index.
	 * <p>Warning: never call this method after wrap policies have been added to tokens
	 * since wrap parent indexes may become invalid.
	 */
	public void remove(int tokenIndex) {
		this.tokens.remove(tokenIndex);
	}

	/**
	 * Adds given token at given index.
	 * <p>Warning: never call this method after wrap policies have been added to tokens
	 * since wrap parent indexes may become invalid.
	 */
	public void insert(int tokenIndex, Token token) {
		this.tokens.add(tokenIndex, token);
	}

	/**
	 * Gets token text with characters escaped as HTML entities where necessary.
	 * @param tokenIndex index of the token to get.
	 */
	public String toString(int tokenIndex) {
		return toString(get(tokenIndex));
	}

	/**
	 * Gets token text with characters escaped as HTML entities where necessary.
	 */
	public String toString(Token token) {
		if (token.isToEscape())
			return getEscapedTokenString(token);
		return token.toString(this.source);
	}

	/**
	 * @return part of the source code defined by given node's position and length.
	 */
	public String toString(ASTNode node) {
		return this.source.substring(node.getStartPosition(), node.getStartPosition() + node.getLength());
	}

	public String getSource() {
		return this.source;
	}

	public int indexOf(Token token) {
		int index = findIndex(token.originalStart, -1, false);
		if (get(index) != token)
			return -1;
		return index;
	}

	public char charAt(int sourcePosition) {
		return this.source.charAt(sourcePosition);
	}

	public int getSourceLength() {
		return this.source.length();
	}

	public int findIndex(int positionInSource, int tokenType, boolean forward) {
		// binary search
		int left = 0, right = size() - 1;
		while (left < right) {
			int index = (right + left) / 2;
			Token token = get(index);
			if (token.originalStart <= positionInSource && positionInSource <= token.originalEnd) {
				left = index;
				break;
			}
			if (token.originalEnd < positionInSource) {
				left = index + 1;
			} else {
				assert token.originalStart > positionInSource;
				right = index - 1;
			}
		}
		int index = left;
		if (!forward && get(index).originalStart > positionInSource)
			index--;
		if (forward && get(index).originalEnd < positionInSource)
			index++;
		while (tokenType >= 0 && get(index).tokenType != tokenType) {
			index += forward ? 1 : -1;
		}
		return index;
	}

	@Override
	public Iterator<Token> iterator() {
		return this.tokens.iterator();
	}

	public boolean isGuardClause(Block node) {
		if (node.statements().size() != 1)
			return false;
		ASTNode parent = node.getParent();
		if (!(parent instanceof IfStatement) || ((IfStatement) parent).getElseStatement() != null)
			return false;
		Object statement = node.statements().get(0);
		if (!(statement instanceof ReturnStatement) && !(statement instanceof ThrowStatement))
			return false;
		// guard clause cannot start with a comment
		// https://bugs.eclipse.org/bugs/show_bug.cgi?id=58565
		int openBraceIndex = firstIndexIn(node, TokenNameLBRACE);
		return !get(openBraceIndex + 1).isComment();
	}

	public int firstIndexIn(ASTNode node, int tokenType) {
		int index = findIndex(node.getStartPosition(), tokenType, true);
		assert tokenInside(node, index);
		return index;
	}

	public Token firstTokenIn(ASTNode node, int tokenType) {
		return get(firstIndexIn(node, tokenType));
	}

	public int lastIndexIn(ASTNode node, int tokenType) {
		int index = findIndex(node.getStartPosition() + node.getLength() - 1, tokenType, false);
		assert tokenInside(node, index);
		return index;
	}

	public Token lastTokenIn(ASTNode node, int tokenType) {
		return get(lastIndexIn(node, tokenType));
	}

	public int firstIndexAfter(ASTNode node, int tokenType) {
		return findIndex(node.getStartPosition() + node.getLength(), tokenType, true);
	}

	public Token firstTokenAfter(ASTNode node, int tokenType) {
		return get(firstIndexAfter(node, tokenType));
	}

	public int firstIndexBefore(ASTNode node, int tokenType) {
		return findIndex(node.getStartPosition() - 1, tokenType, false);
	}

	public Token firstTokenBefore(ASTNode node, int tokenType) {
		return get(firstIndexBefore(node, tokenType));
	}

	public int countLineBreaksBetween(Token previous, Token current) {
		int start = previous != null ? previous.originalEnd + 1 : 0;
		int end = current != null ? current.originalStart : this.source.length();
		return countLineBreaksBetween(this.source, start, end);
	}

	public int countLineBreaksBetween(String text, int startPosition, int endPosition) {
		int result = 0;
		for (int i = startPosition; i < endPosition; i++) {
			switch (text.charAt(i)) {
				case '\r':
					result++;
					if (i + 1 < endPosition && text.charAt(i + 1) == '\n')
						i++;
					break;
				case '\n':
					result++;
					if (i + 1 < endPosition && text.charAt(i + 1) == '\r')
						i++;
					break;
			}
		}
		return result;
	}

	private TokenTraverser positionInLineCounter = new TokenTraverser() {
		private boolean isNLSTagInLine = false;

		@Override
		protected boolean token(Token traversed, int index) {
			if (index == this.value) {
				this.isNLSTagInLine = false;
				return false;
			}
			if (traversed.hasNLSTag()) {
				assert traversed.tokenType == TokenNameStringLiteral;
				this.isNLSTagInLine = true;
			}
			if (traversed.getAlign() > 0)
				this.counter = traversed.getAlign();
			List<Token> internalStructure = traversed.getInternalStructure();
			if (internalStructure != null && !internalStructure.isEmpty()) {
				assert traversed.tokenType == TokenNameCOMMENT_BLOCK || traversed.tokenType == TokenNameCOMMENT_JAVADOC;
				this.counter = TokenManager.this.commentWrapper.wrapMultiLineComment(traversed, this.counter, true,
						this.isNLSTagInLine);
			} else {
				this.counter += getLength(traversed, this.counter);
			}
			if (isSpaceAfter())
				this.counter++;
			return true;
		}
	};

	public int getPositionInLine(int tokenIndex) {
		Token token = get(tokenIndex);
		if (token.getAlign() > 0)
			return get(tokenIndex).getAlign();
		// find the first token in line and calculate position of given token
		int firstTokenIndex = token.getLineBreaksBefore() > 0 ? tokenIndex : findFirstTokenInLine(tokenIndex);
		Token firstToken = get(firstTokenIndex);
		int startingPosition = toIndent(firstToken.getIndent(), firstToken.getWrapPolicy() != null);
		if (firstTokenIndex == tokenIndex)
			return startingPosition;

		this.positionInLineCounter.value = tokenIndex;
		this.positionInLineCounter.counter = startingPosition;
		traverse(firstTokenIndex, this.positionInLineCounter);
		return this.positionInLineCounter.counter;
	}

	public int findSourcePositionInLine(int position) {
		int lineStartPosition = position;
		char c;
		while (lineStartPosition > 0 && (c = charAt(lineStartPosition)) != '\r' && c != '\n')
			lineStartPosition--;
		int positionInLine = getLength(lineStartPosition, position - 1, 0);
		return positionInLine;
	}

	private String getEscapedTokenString(Token token) {
		if (token.getLineBreaksBefore() > 0 && charAt(token.originalStart) == '@') {
			return "@" + this.source.substring(token.originalStart + 1, token.originalEnd + 1); //$NON-NLS-1$
		} else if (token.tokenType == TokenNameNotAToken) {
			String text = token.toString(this.source);
			Matcher matcher = COMMENT_LINE_ANNOTATION_PATTERN.matcher(text);
			if (matcher.find()) {
				return matcher.group(1) + "@" + text.substring(matcher.end(2)); //$NON-NLS-1$
			}
		}
		return token.toString(this.source);
	}

	/**
	 * @param token the token to measure
	 * @param startPosition position in line of the first character (affects tabs calculation)
	 * @return actual length of given token, considering tabs and escaping characters as HTML entities
	 */
	public int getLength(Token token, int startPosition) {
		int length = getLength(token.originalStart, token.originalEnd, startPosition);
		if (token.isToEscape()) {
			if (token.getLineBreaksBefore() > 0 && charAt(token.originalStart) == '@') {
				length += 4; // 4 = "@".length() - "@".length()
			} else if (token.tokenType == TokenNameNotAToken) {
				Matcher matcher = COMMENT_LINE_ANNOTATION_PATTERN.matcher(token.toString(this.source));
				if (matcher.find()) {
					length += 4; // 4 = "@".length() - "@".length()
				}
			}
		}
		return length;
	}

	/**
	 * Calculates the length of a source code fragment.
	 * @param originalStart the first position of the source code fragment
	 * @param originalEnd the last position of the source code fragment 
	 * @param startPosition position in line of the first character (affects tabs calculation)
	 * @return length, considering tabs and escaping characters as HTML entities
	 */
	public int getLength(int originalStart, int originalEnd, int startPosition) {
		int position = startPosition;
		for (int i = originalStart; i <= originalEnd; i++) {
			switch (this.source.charAt(i)) {
				case '\t':
					if (this.tabSize > 0)
						position += this.tabSize - position % this.tabSize;
					break;
				case '\r':
				case '\n':
					position = 0;
					break;
				default:
					position++;
			}
		}
		return position - startPosition;
	}

	/**
	 * @param indent desired indentation (in positions, not in levels)
	 * @param isWrapped whether indented element is wrapped
	 * @return actual indentation that can be achieved with current settings
	 */
	public int toIndent(int indent, boolean isWrapped) {
		if (this.tabChar == DefaultCodeFormatterOptions.TAB && !(isWrapped && this.wrapWithSpaces)) {
			int tab = this.tabSize;
			if (tab <= 0)
				return 0;
			indent = ((indent + tab - 1) / tab) * tab;
		}
		return indent;
	}

	public int traverse(int startIndex, TokenTraverser traverser) {
		return traverser.traverse(this.tokens, startIndex);
	}

	public int findFirstTokenInLine(int startIndex) {
		return findFirstTokenInLine(startIndex, false, false);
	}

	public int findFirstTokenInLine(int startIndex, boolean includeWraps, boolean includeIndents) {
		Token previous = get(startIndex); // going backwards, previous has higher index than current
		for (int i = startIndex - 1; i >= 0; i--) {
			Token token = get(i);
			if (token.getLineBreaksAfter() > 0 || previous.getLineBreaksBefore() > 0) {
				boolean include = previous.getWrapPolicy() != null
						&& (previous.getWrapPolicy().wrapMode == WrapMode.BLOCK_INDENT ? includeIndents : includeWraps);
				if (!include)
					return i + 1;
			}
			previous = token;
		}
		return 0;
	}

	private boolean tokenInside(ASTNode node, int index) {
		return get(index).originalStart >= node.getStartPosition()
				&& get(index).originalEnd <= node.getStartPosition() + node.getLength();
	}

	public void addNLSAlignIndex(int index, int align) {
		if (this.tokenIndexToNLSAlign == null)
			this.tokenIndexToNLSAlign = new HashMap<Integer, Integer>();
		this.tokenIndexToNLSAlign.put(index, align);
	}

	public int getNLSAlign(int index) {
		if (this.tokenIndexToNLSAlign == null)
			return 0;
		Integer align = this.tokenIndexToNLSAlign.get(index);
		return align != null ? align : 0;
	}

	public void setHeaderEndIndex(int headerEndIndex) {
		this.headerEndIndex = headerEndIndex;
	}

	public boolean isInHeader(int tokenIndex) {
		return tokenIndex < this.headerEndIndex;
	}

	public void addDisableFormatTokenPair(Token formatOffTag, Token formatOnTag) {
		if (this.formatOffTagPairs == null)
			this.formatOffTagPairs = new ArrayList<Token[]>();
		this.formatOffTagPairs.add(new Token[] { formatOffTag, formatOnTag });
	}

	public void applyFormatOff() {
		if (this.formatOffTagPairs == null)
			return;
		for (Token[] pair : this.formatOffTagPairs) {
			int index1 = findIndex(pair[0].originalStart, -1, false);
			int index2 = findIndex(pair[1].originalEnd, -1, false);
			pair[0] = get(index1);
			pair[1] = get(index2);
			Token unformatted = new Token(pair[0].originalStart, pair[1].originalEnd, TokenNameWHITESPACE);
			unformatted.setIndent(Math.min(pair[0].getIndent(), findSourcePositionInLine(pair[0].originalStart)));
			unformatted.putLineBreaksBefore(pair[0].getLineBreaksBefore());
			if (pair[0].isSpaceBefore())
				unformatted.spaceBefore();
			unformatted.putLineBreaksAfter(pair[1].getLineBreaksAfter());
			if (pair[1].isSpaceAfter())
				unformatted.spaceAfter();
			this.tokens.set(index1, unformatted);
			this.tokens.subList(index1 + 1, index2 + 1).clear();
		}
	}
}