/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hive.ptest.execution.conf;
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.base.Supplier;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
public class TestParser {
private static final Splitter TEST_SPLITTER = Splitter.onPattern("[, ]")
.trimResults().omitEmptyStrings();
private static final String QTEST_MODULE_NAME = "itests/qtest";
private static final String QTEST_SPARK_MODULE_NAME = "itests/qtest-spark";
private final AtomicInteger batchIdCounter;
private final Context context;
private final String testCasePropertyName;
private final File sourceDirectory;
private final Logger logger;
public TestParser(Context context, AtomicInteger batchIdCounter, String testCasePropertyName,
File sourceDirectory, Logger logger) {
this.context = context;
this.batchIdCounter = batchIdCounter;
this.testCasePropertyName = testCasePropertyName;
this.sourceDirectory = sourceDirectory;
this.logger = logger;
}
private List<TestBatch> parseTests() {
Set<String> excluded = new HashSet<String>();
List<TestBatch> result = Lists.newArrayList();
for(QFileTestBatch test : parseQFileTests()) {
result.add(test);
excluded.add(test.getDriver());
}
Collection<TestBatch> unitTestBatches =
new UnitTestPropertiesParser(context, batchIdCounter, testCasePropertyName, sourceDirectory, logger,
excluded).generateTestBatches();
result.addAll(unitTestBatches);
return result;
}
private List<QFileTestBatch> parseQFileTests() {
Map<String, Properties> properties = parseQTestProperties();
List<QFileTestBatch> result = Lists.newArrayList();
String qFileTestsString = context.getString("qFileTests",null);
String []aliases;
if (qFileTestsString != null) {
aliases = qFileTestsString.split(" ");
} else {
aliases = new String[0];
}
for(String alias : aliases) {
Context testContext = new Context(context.getSubProperties(
Joiner.on(".").join("qFileTest", alias, "")));
String driver = checkNotNull(testContext.getString("driver"), "driver").trim();
// execute the driver locally?
boolean isParallel = !testContext.getBoolean("isolateDriver", false);
File directory = new File(sourceDirectory,
checkNotNull(testContext.getString("directory"), "directory").trim());
Set<String> excludedTests = Sets.newHashSet();
for(String excludedTestGroup : TEST_SPLITTER.split(testContext.getString("exclude", ""))) {
excludedTests.addAll(Arrays.asList(testContext.
getString(Joiner.on(".").join("groups", excludedTestGroup), "").trim().split(" ")));
expandTestProperties(excludedTests, properties);
}
Set<String> isolatedTests = Sets.newHashSet();
for(String ioslatedTestGroup : TEST_SPLITTER.split(testContext.getString("isolate", ""))) {
isolatedTests.addAll(Arrays.asList(testContext.
getString(Joiner.on(".").join("groups", ioslatedTestGroup), "").trim().split(" ")));
expandTestProperties(isolatedTests, properties);
}
Set<String> includedTests = Sets.newHashSet();
for(String includedTestGroup : TEST_SPLITTER.split(testContext.getString("include", ""))) {
includedTests.addAll(Arrays.asList(testContext.
getString(Joiner.on(".").join("groups", includedTestGroup), "").trim().split(" ")));
expandTestProperties(includedTests, properties);
}
//excluded overrides included
includedTests.removeAll(excludedTests);
result.addAll(createQFileTestBatches(
driver,
checkNotNull(testContext.getString("queryFilesProperty"), "queryFilesProperty").trim(),
directory,
testContext.getInteger("batchSize", 30),
isParallel,
excludedTests,
includedTests,
isolatedTests));
}
return result;
}
private List<QFileTestBatch> createQFileTestBatches(String driver, String queryFilesProperty,
File directory, int batchSize, boolean isParallel, Set<String> excluded,
Set<String> included, Set<String> isolated) {
logger.info("Create batches for " + driver);
List<String> qFileTestNames = Lists.newArrayList();
for(File test : checkNotNull(directory.listFiles(), directory.getAbsolutePath())) {
String testName = test.getName();
if(test.isFile() &&
testName.endsWith(".q") &&
(included.isEmpty() || included.contains(testName))) {
qFileTestNames.add(testName);
}
}
List<QFileTestBatch> testBatches = Lists.newArrayList();
List<String> testBatch = Lists.newArrayList();
for(final String test : qFileTestNames) {
if(excluded.contains(test)) {
logger.info("Exlcuding test " + driver + " " + test);
} else if(isolated.contains(test)) {
logger.info("Executing isolated test " + driver + " " + test);
testBatches.add(new QFileTestBatch(batchIdCounter, testCasePropertyName, driver, queryFilesProperty,
Sets.newHashSet(test), isParallel, getModuleName(driver)));
} else {
if(testBatch.size() >= batchSize) {
testBatches.add(new QFileTestBatch(batchIdCounter, testCasePropertyName, driver, queryFilesProperty,
Sets.newHashSet(testBatch), isParallel, getModuleName(driver)));
testBatch = Lists.newArrayList();
}
testBatch.add(test);
}
}
if(!testBatch.isEmpty()) {
testBatches.add(new QFileTestBatch(batchIdCounter, testCasePropertyName, driver, queryFilesProperty,
Sets.newHashSet(testBatch), isParallel, getModuleName(driver)));
}
return testBatches;
}
/**
* @return properties loaded from files specified in qFileTests.propertyFiles.${fileName}=${filePath}
*/
private Map<String, Properties> parseQTestProperties() {
Map<String, String> propFiles = context.getSubProperties("qFileTests.propertyFiles.");
Map<String, Properties> propertyMap = new HashMap<String, Properties>();
for (String propFile : propFiles.keySet()) {
Properties properties = new Properties();
String path = sourceDirectory + File.separator + propFiles.get(propFile);
FileInputStream fis = null;
try {
fis = new FileInputStream(path);
properties.load(fis);
} catch (IOException e) {
logger.warn("Error processing Qtest property file", e);
throw new IllegalArgumentException("Error processing Qtest property file: " + path);
} finally {
try {
if (fis != null) {
fis.close();
}
} catch (IOException e) { //ignore
}
}
propertyMap.put(propFile, properties);
logger.info("Loaded Qtest property file: " + path);
}
return propertyMap;
}
/**
* If any of given tests are of the form: ${fileName}.${property} (test list within a property file),
* then expand them. Then remove those markers from the list of tests.
*/
private void expandTestProperties(Set<String> tests, Map<String, Properties> propMap) {
Set<String> toRemove = new HashSet<String>();
Set<String> toAdd = new HashSet<String>();
String pattern = "([^\\.]*)\\.\\$\\{([^}]*)}";
Pattern r = Pattern.compile(pattern);
for (String test : tests) {
Matcher m = r.matcher(test);
if (m.find()) {
toRemove.add(test);
logger.info("Expanding qfile property: " + test);
String propName = m.group(1);
String propValue = m.group(2);
Properties props = propMap.get(propName);
if (props == null) {
logger.warn("No properties found for : " + propName);
throw new IllegalArgumentException("No properties found for : " + propName);
}
String result = (String) props.get(propValue);
if (result == null || result.isEmpty()) {
logger.warn("No properties found in file: " + propName + " for property: " + propValue);
throw new IllegalArgumentException("No propertifies found in file: " + propName + " for property: " + propValue);
}
Iterable<String> splits = TEST_SPLITTER.split(result);
for (String split : splits) {
toAdd.add(split);
}
}
}
tests.removeAll(toRemove);
tests.addAll(toAdd);
}
private String getModuleName(String driverName) {
if (driverName.toLowerCase().contains("spark")) {
return QTEST_SPARK_MODULE_NAME;
} else {
return QTEST_MODULE_NAME;
}
}
public Supplier<List<TestBatch>> parse() {
return new Supplier<List<TestBatch>>() {
@Override
public List<TestBatch> get() {
return parseTests();
}
};
}
/**
* Manually test this against any property file.
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
if (args.length < 1) {
throw new IllegalArgumentException("Enter the property file location");
}
Logger log = LoggerFactory
.getLogger(TestParser.class);
File workingDir = new File("../..");
File testConfigurationFile = new File(args[0]);
TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile, log);
TestParser testParser = new TestParser(conf.getContext(), new AtomicInteger(1), "test", workingDir, log);
List<TestBatch> testBatches = testParser.parse().get();
for (TestBatch testBatch : testBatches) {
System.out.println(testBatch.getTestArguments());
}
}
}