AbstractLexerTest.java

/*******************************************************************************
 * Copyright (c) 2012 Bruno Medeiros and other Contributors.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     Bruno Medeiros - initial API and implementation
 *******************************************************************************/
package dtool.parser.common;


import static melnorme.utilbox.core.Assert.AssertNamespace.assertTrue;
import dtool.parser.DeeLexer;
import dtool.parser.DeeTokens;
import dtool.parser.DeeLexerErrors;
import dtool.tests.CommonDToolTest;

public class AbstractLexerTest extends CommonDToolTest {
	
	public AbstractLexerTest() {
	}
	
	public static DeeLexer testLexerTokenizing(String source, DeeTokens... deeTokens) {
		return testLexerTokenizing(source, tokenCheckers(deeTokens));
	}
	
	public static TokenChecker[] tokenCheckers(DeeTokens... deeTokens) {
		TokenChecker[] tokenCheckers = new TokenChecker[deeTokens.length];
		for (int i = 0; i < deeTokens.length; i++) {
			tokenCheckers[i] = new TokenChecker(deeTokens[i]);
		}
		return tokenCheckers;
	}
	
	public static DeeLexer testLexerTokenizing(String source, TokenChecker... tokenCheckers) {
		DeeLexer deeLexer = new DeeTestsLexer(source);
		testLexerTokenizing(deeLexer, 0, tokenCheckers);
		return deeLexer;
	}
	public static void testLexerTokenizing(DeeLexer deeLexer, int readSourceOffset, TokenChecker... tokenCheckers) {
		String source = deeLexer.source;
		
		StringBuilder constructedSource = new StringBuilder(deeLexer.source.substring(0, readSourceOffset));
		for (int i = 0; i < tokenCheckers.length; i++) {
			TokenChecker tokenChecker = tokenCheckers[i];
			Token token = tokenChecker.checkToken(deeLexer, readSourceOffset);
			readSourceOffset = token.getEndPos();
			String sourceSoFar = source.substring(0, readSourceOffset);
			
			String tokenSourceValue = token.getSourceValue();
			// retest with just the token source to make sure boundaries are correct
			if(tokenCheckers.length != 1 && !tokenSourceValue.startsWith("#!")) {
				testLexerTokenizing(tokenSourceValue, array(tokenChecker));
			}
			
			constructedSource.append(tokenSourceValue);
			assertTrue(sourceSoFar.contentEquals(constructedSource));
		}
		assertTrue(deeLexer.pos == source.length());
		new TokenChecker(DeeTokens.EOF).checkToken(deeLexer, readSourceOffset);
		assertTrue(deeLexer.tokenStartPos == source.length());
		assertEquals(source, constructedSource.toString());
	}
	
	protected static final class DeeTestsLexer extends DeeLexer {
		public DeeTestsLexer(String source) {
			super(source);
		}
		
		@Override
		public String toString() {
			return source.substring(0, pos) + "<---parser--->" + source.substring(pos, source.length());
		}
	}
	
	public static class TokenChecker {
		
		private DeeTokens expectedTokenType;
		private DeeLexerErrors expectedError;
		
		public TokenChecker(DeeTokens deeToken) {
			this(deeToken, null);
		}
		
		public TokenChecker(DeeTokens tokenType, DeeLexerErrors error) {
			this.expectedTokenType = tokenType;
			this.expectedError = error;
			if(tokenType == DeeTokens.INVALID_TOKEN) {
				assertTrue(expectedError == null);
				this.expectedError = DeeLexerErrors.INVALID_CHARACTERS;
			}
		}
		
		public Token checkToken(DeeLexer deeLexer, int readOffset) {
			Token token = deeLexer.next();
			
			if(expectedTokenType != null) {
				assertTrue(token.type == expectedTokenType);
			} else {
				assertTrue(expectedError == null);
			}
			
			assertTrue(token.getError() == expectedError);
			
			assertTrue(token.getStartPos() == readOffset);
			assertEquals(deeLexer.source.subSequence(token.getStartPos(), token.getEndPos()), token.getSourceValue());
			
			DeeTokens tokenCode = token.getType();
			if(tokenCode == DeeTokens.EOF) {
				assertTrue(token.getEndPos() >= token.getStartPos());
			} else {
				assertTrue(token.getEndPos() > token.getStartPos());
			}
			return token;
		}
	}
	
}