DeeParserSourceTests.java

/*******************************************************************************
 * Copyright (c) 2012, 2012 IBM Corporation and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     Bruno Medeiros - initial API and implementation
 *******************************************************************************/
package dtool.parser;

import static dtool.util.NewUtils.isValidStringRange;
import static dtool.util.NewUtils.replaceRange;
import static dtool.util.NewUtils.substringRemoveEnd;
import static melnorme.utilbox.core.Assert.AssertNamespace.assertFail;
import static melnorme.utilbox.core.Assert.AssertNamespace.assertNotNull;
import static melnorme.utilbox.core.Assert.AssertNamespace.assertTrue;
import static melnorme.utilbox.core.CoreUtil.areEqual;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;

import dtool.parser.DeeParserTester.NamedNodeElement;
import dtool.parser.common.IToken;
import dtool.parser.common.LexElement;
import dtool.parser.common.LexElementSource;
import dtool.sourcegen.AnnotatedSource;
import dtool.sourcegen.AnnotatedSource.MetadataEntry;
import dtool.sourcegen.TemplatedSourceProcessorParser.TspExpansionElement;
import dtool.tests.CommonTemplatedSourceBasedTest;
import dtool.tests.utils.SimpleParser;
import dtool.util.NewUtils;
import melnorme.lang.tooling.ast.ParserErrorTypes;
import melnorme.lang.tooling.ast.SourceRange;
import melnorme.lang.tooling.common.ParserError;
import melnorme.utilbox.collections.ArrayList2;
import melnorme.utilbox.concurrency.OperationCancellation;
import melnorme.utilbox.misc.ArrayUtil;
import melnorme.utilbox.misc.Pair;
import melnorme.utilbox.misc.StringUtil;

@RunWith(Parameterized.class)
public class DeeParserSourceTests extends CommonTemplatedSourceBasedTest {
	
	public static final String TESTFILESDIR = "parser";
	
	protected static Map<String, TspExpansionElement> commonDefinitions = new HashMap<>();
	
	@BeforeClass
	public static void initCommonDefinitions() throws IOException {
		addCommonDefinitions(DeeParserSourceTests.class, TESTFILESDIR, commonDefinitions);
	}
	
	@Parameters(name="{index}: {0}")
	public static Collection<Object[]> testFilesList() throws IOException {
		return createTestFileParameters(TESTFILESDIR);
	}
	
	public DeeParserSourceTests(String testUIDescription, File file) {
		super(testUIDescription, file);
	}
	
	@Test
	public void runSourceBasedTests() throws Exception { runSourceBasedTests$(); }
	public void runSourceBasedTests$() throws Exception {
		runAnnotatedTests(getTestCasesFromFile(commonDefinitions));
	}
	
	@Override
	public void runAnnotatedSourceTest(AnnotatedSource testSource) {
		final String DEFAULT_VALUE = "##DEFAULT VALUE";
		
		String fullSource = testSource.source;
		final LexElementSource lexSource = new DeeParser(fullSource).getEnabledLexSource();
		
		String expectedParsedSource = fullSource;
		String expectedRemainingSource = null;
		String parseRule = null;
		
		String expectedPrintedSource = DEFAULT_VALUE;
		NamedNodeElement[] expectedStructure = null;
		boolean allowAnyErrors = false;
		boolean ignoreFurtherErrorMDs = false;
		
		ArrayList2<ParserError> expectedErrors = new ArrayList2<>();
		List<MetadataEntry> additionalMetadata = new ArrayList<>();
		List<StringCorrection> errorCorrectionMetadata = new ArrayList<>();
		
		for (MetadataEntry mde : testSource.metadata) {
			if(mde.name.equals("AST_SOURCE_EXPECTED")) {
				assertTrue(expectedPrintedSource == DEFAULT_VALUE);
				if(areEqual(mde.value, "NoCheck")) {
					expectedPrintedSource = null;
				} else {
					expectedPrintedSource = assertNotNull(mde.sourceValue);
				}
				ignoreFurtherErrorMDs = true;
			} else if(mde.name.equals("PARSE")){
				parseRule = mde.value;
			} else if(mde.name.equals("parser") && areEqual(mde.value, "CutRest")){
				int pos = assertNoLength(mde).offset;
				
				if(expectedRemainingSource == null) {
					fullSource = fullSource.substring(0, pos);
					expectedParsedSource = fullSource;
					expectedRemainingSource = "";
				} else {
					int endLengthToRemove = fullSource.length() - pos;
					fullSource = fullSource.substring(0, pos);
					expectedRemainingSource = substringRemoveEnd(expectedRemainingSource, endLengthToRemove);
				}
				ignoreFurtherErrorMDs = true;
				
			} else if(mde.name.equals("parser") && areEqual(mde.value, "IgnoreRest")){
				int pos = assertNoLength(mde).offset;
				
				if(expectedRemainingSource == null) {
					expectedParsedSource = fullSource.substring(0, pos);
					expectedRemainingSource = fullSource.substring(pos);
					ignoreFurtherErrorMDs = true;
				}
			} else if(mde.name.equals("STRUCTURE_EXPECTED")) {
				assertTrue(expectedStructure == null);
				expectedStructure = parseExpectedStructure(mde.sourceValue);
			} else if(mde.name.equals("error") || mde.name.equals("ERROR")){
				if(areEqual(mde.value, "-none-")) {
					int offset = ignoreFurtherErrorMDs ? expectedParsedSource.length() : mde.offset;
						
					errorCorrectionMetadata.add(new StringCorrection(offset, 0, mde.sourceValue));
					continue;
				}
				if(ignoreFurtherErrorMDs) 
					continue;
				
				ParserError error = decodeError(lexSource, mde);
				expectedErrors.add(error);
				
				if(getErrorTypeFromMDE(mde) == ParserErrorTypes.INVALID_TOKEN_CHARACTERS) {
					SourceRange sr = error.sourceRange;
					errorCorrectionMetadata.add(new StringCorrection(sr.getOffset(), sr.getLength(), ""));
				} else if(getErrorTypeFromMDE(mde) == ParserErrorTypes.EXPECTED_TOKEN) {
					assertTrue(mde.sourceValue == null || mde.sourceValue.isEmpty() || !mde.sourceWasIncluded);
					
					String rpl = mde.sourceValue;
					if(rpl == null) {
						DeeTokens expectedToken = (DeeTokens) error.msgData;
						if(!expectedToken.hasSourceValue()) 
							continue;
						rpl = expectedToken.getSourceValue();
					}
					
					errorCorrectionMetadata.add(new StringCorrection(error.sourceRange.getEndPos(), 0, rpl));
				}
				
			} else if(mde.name.equals("parser") && areEqual(mde.value, "AllowAnyErrors")){
				allowAnyErrors = true;
			} else if(mde.name.equals("test") && areEqual(mde.value, "IGNORE_REMAINING_SOURCE_CHECK")){
				expectedRemainingSource = DeeParserTester.DONT_CHECK;
			} else if(areEqual(mde.value, "test")){
				additionalMetadata.add(mde);
			} else {
				if(!(areEqual(mde.value, "flag") || areEqual(mde.name, "comment"))) {
					assertFail("Unknown metadata");
				}
			}
		}
		
		// Do error correction for toStringAsCode
		if(expectedPrintedSource == DEFAULT_VALUE) {
			expectedPrintedSource = calcExpectedToStringAsCode(expectedParsedSource, errorCorrectionMetadata);
		}
		
		if(allowAnyErrors) {
			expectedErrors = null;
		}
		
		try {
			new DeeParserTester(fullSource, parseRule, expectedRemainingSource, expectedPrintedSource, 
				expectedStructure, expectedErrors, additionalMetadata).runParserTest______________________();
		} catch(OperationCancellation e) {
			throw assertFail();
		}
	}
	
	public static MetadataEntry assertNoLength(MetadataEntry mde) {
		assertTrue(mde.offset >= 0);
		assertTrue(mde.sourceValue == null);
		return mde;
	}
	
	public static class StringCorrection {
		public String rpl;
		public int offset;
		public int length;
		
		public StringCorrection(int offset, int length, String rpl) {
			this.rpl = rpl;
			this.offset = offset;
			this.length = length;
		}
		
	}
	
	public static String calcExpectedToStringAsCode(String parseSource, List<StringCorrection> errorCorrections) {
		int modifyDelta = 0;
		
		String correctedParseSource = parseSource;
		for (StringCorrection sc : errorCorrections) {
			
			int offset = sc.offset + modifyDelta;
			assertTrue(isValidStringRange(correctedParseSource, offset, sc.length));
			correctedParseSource = replaceRange(correctedParseSource, offset , sc.length, sc.rpl);
			modifyDelta += sc.rpl.length() - sc.length; // can be negative
		}
		return correctedParseSource;
	}
	
	protected static final Map<String,ParserErrorTypes> errorNameToType = NewUtils.newHashMap(
		Pair.create("ITC", ParserErrorTypes.INVALID_TOKEN_CHARACTERS),
		Pair.create("MT", ParserErrorTypes.MALFORMED_TOKEN),
		Pair.create("MTC", ParserErrorTypes.MALFORMED_TOKEN),
		Pair.create("EXP", ParserErrorTypes.EXPECTED_TOKEN),
		Pair.create("EXPRULE", ParserErrorTypes.EXPECTED_RULE),
		Pair.create("SE", ParserErrorTypes.SYNTAX_ERROR),
		Pair.create("<SE", ParserErrorTypes.SYNTAX_ERROR),
		Pair.create("REQPARENS", ParserErrorTypes.EXP_MUST_HAVE_PARENTHESES),
		Pair.create("TYPE_AS_EXP_VALUE", ParserErrorTypes.TYPE_USED_AS_EXP_VALUE),
		Pair.create("NO_TPL_SINGLE_ARG", ParserErrorTypes.NO_CHAINED_TPL_SINGLE_ARG),
		Pair.create("BAD_LINKAGE_ID", ParserErrorTypes.INVALID_EXTERN_ID)
	);
	
	public static ParserErrorTypes getErrorTypeFromMDE(MetadataEntry mde) {
		try {
			return ParserErrorTypes.valueOf(mde.value);
		} catch (IllegalArgumentException e) {
			// continue
		}
		
		ParserErrorTypes result = errorNameToType.get(mde.value);
		if(result != null) {
			return result;
		}
		String errorType = StringUtil.substringUntilMatch(mde.value, "_");
		return assertNotNull(errorNameToType.get(errorType));
	}
	
	public static ParserError decodeError(LexElementSource lexSource, MetadataEntry mde) {
		String errorTypeStr = StringUtil.substringUntilMatch(mde.value, "_");
		String errorParam = StringUtil.segmentAfterMatch(mde.value, "_");
		
		SourceRange errorRange = mde.getSourceRange();
		String errorSource = null;
		
		ParserErrorTypes errorType = getErrorTypeFromMDE(mde);
		switch (errorType) {
		case INVALID_TOKEN_CHARACTERS:
			return new ParserError(ParserErrorTypes.INVALID_TOKEN_CHARACTERS, errorRange, mde.sourceValue, null);
		case MALFORMED_TOKEN:
			errorParam = DeeLexerSourceTests.parseExpectedError(errorParam).toString();
			return createErrorToken(ParserErrorTypes.MALFORMED_TOKEN, mde, lexSource, true, errorParam);
		case EXPECTED_TOKEN:
			String expectedTokenStr = DeeLexerSourceTests.transformTokenNameAliases(errorParam);
			DeeTokens expectedToken = DeeTokens.valueOf(expectedTokenStr);
			return createErrorToken(ParserErrorTypes.EXPECTED_TOKEN, mde, lexSource, true, expectedToken);
		case EXPECTED_RULE:
			errorParam = getExpectedRuleDescription(errorParam);
			return createErrorToken(ParserErrorTypes.EXPECTED_RULE, mde, lexSource, true, errorParam);
		case SYNTAX_ERROR:
			errorParam = getExpectedRuleDescription(errorParam);
			boolean tokenBefore = errorTypeStr.equals("<SE");
			return createErrorToken(ParserErrorTypes.SYNTAX_ERROR, mde, lexSource, tokenBefore, errorParam);
		case EXP_MUST_HAVE_PARENTHESES: 
			errorParam = errorParam == null ? DeeParserTester.DONT_CHECK : errorParam;
			errorSource = assertNotNull(mde.sourceValue);
			return new ParserError(errorType, errorRange, errorSource, errorParam);
		case TYPE_USED_AS_EXP_VALUE:
		case INIT_USED_IN_EXP:
		case NO_CHAINED_TPL_SINGLE_ARG:
			errorSource = assertNotNull(mde.sourceValue);
			return new ParserError(errorType, errorRange, errorSource, null);
		case INVALID_EXTERN_ID:
		case INVALID_SCOPE_ID:
		case INVALID_TRAITS_ID:
			if(mde.sourceValue != null) {
				return new ParserError(errorType, errorRange, mde.sourceValue, null);
			}
			return createErrorToken(errorType, mde, lexSource, true, null);
		case LAST_CATCH:
			return createErrorToken(ParserErrorTypes.LAST_CATCH, mde, lexSource, false, null);
		}
		throw assertFail();
	}
	
	public static ParserError createErrorToken(ParserErrorTypes errorTypeTk, MetadataEntry mde, 
		LexElementSource lexSource, boolean tokenBefore, Object errorParam) {
		IToken adjacentToken = tokenBefore 
			? findLastEffectiveTokenBeforeOffset(mde.offset, lexSource)
			: findNextEffectiveTokenAfterOffset(mde.offset, lexSource);
			
		SourceRange errorRange = adjacentToken.getSourceRange();
		String errorSource = adjacentToken.getSourceValue();
		return new ParserError(errorTypeTk, errorRange, errorSource, errorParam);
	}
	
	public static String getExpectedRuleDescription(String ruleId) {
		if(ruleId.equals("decl")) {
			return DeeParser.RULE_DECLARATION.description;
		} else if(ruleId.equals("exp")) {
			return DeeParser.RULE_EXPRESSION.description;
		} else if(ruleId.equals("ref")) {
			return DeeParser.RULE_REFERENCE.description;
		}
		
		assertTrue(DeeParser.getRule(ruleId) != null);
		return DeeParser.getRule(ruleId).description;
	}
	
	public static LexElement findLastEffectiveTokenBeforeOffset(int offset, LexElementSource lexSource) {
		List<LexElement> lexElementList = lexSource.getLexElementList();
		assertTrue(offset > 0 && offset <= lexElementList.get(lexElementList.size()-1).getEndPos());
		
		LexElement lastLexElement = LexElementSource.START_TOKEN;
		for (LexElement lexElement : lexElementList) {
			if(lexElement.getStartPos() >= offset)
				break;
			lastLexElement = lexElement;
		}
		return lastLexElement == null ? null : lastLexElement;
	}
	
	public static LexElement findNextEffectiveTokenAfterOffset(int offset, LexElementSource lexSource) {
		List<LexElement> lexElementList = lexSource.getLexElementList();
		
		for (LexElement lexElement : lexElementList) {
			if(lexElement.isEOF()) {
				assertFail();
			}
			if(lexElement.getStartPos() >= offset) {
				return lexElement;
			}
		}
		throw assertFail();
	}
	
	protected NamedNodeElement[] parseExpectedStructure(String source) {
		SimpleParser parser = new SimpleParser(source);
		NamedNodeElement[] namedElements = readNamedElementsList(parser);
		assertTrue(parser.lookaheadIsEOF() || parser.lookAhead() == '$');
		return namedElements;
	}
	
	public static NamedNodeElement[] readNamedElementsList(SimpleParser parser) {
		ArrayList<NamedNodeElement> elements = new ArrayList<NamedNodeElement>();
		
		while(true) {
			String id;
			NamedNodeElement[] children = null;
			
			parser.seekWhiteSpace();
			if(parser.tryConsume("*")) {
				id = NamedNodeElement.IGNORE_ALL;
			} else {
				if(parser.tryConsume("?")) {
					id = NamedNodeElement.IGNORE_NAME;
				} else {
					id = parser.consumeAlphaNumericUS(true);
					if(id.isEmpty()) {
						break;
					}
					parser.seekWhiteSpace();
				}
				if(parser.tryConsume("(")) {
					children = readNamedElementsList(parser);
					parser.seekWhiteSpace();
					assertTrue(parser.tryConsume(")") || parser.lookAhead() == '$');
				} else {
					children = new NamedNodeElement[0];
				}
			}
			elements.add(new NamedNodeElement(id, children));
		}
		return ArrayUtil.createFrom(elements, NamedNodeElement.class);
	}

}