/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.internal.processors.hadoop.HadoopTestClassLoader;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopTeraSortTest;
import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolEmbeddedSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolMultipleServersSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopTxConfigCacheTest;
import org.apache.ignite.internal.processors.hadoop.impl.fs.KerberosHadoopFileSystemFactorySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.Hadoop1OverIgfsProxyTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemClientBasedDualAsyncSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemClientBasedDualSyncSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemClientBasedOpenTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemClientBasedPrimarySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemClientBasedProxySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalToClientDualAsyncSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalToClientDualSyncSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalToClientPrimarySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalToClientProxySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.util.BasicUserNameMapperSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.util.ChainedUserNameMapperSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.util.KerberosUserNameMapperSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.Hadoop1OverIgfsDualAsyncTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.Hadoop1OverIgfsDualSyncTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopFIleSystemFactorySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfs20FileSystemLoopbackPrimarySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsDualAsyncSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsDualSyncSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopSecondaryFileSystemConfigurationTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgfsEventsTestSuite;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemClientSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemHandshakeSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoggerSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoggerStateSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopCommandLineTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopFileSystemsTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopGroupingTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopJobTrackerSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopMapReduceEmbeddedSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopMapReduceErrorResilienceTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopMapReduceTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopNoHadoopMapReduceTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopSerializationWrapperSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopSnappyFullMapReduceTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopSnappyTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopSortingTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopSplitWrapperSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopTaskExecutionSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopTasksV1Test;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopTasksV2Test;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopUserLibsSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopV2JobSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopValidationSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopWeightedMapReducePlannerTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopWeightedPlannerMapReduceTest;
import org.apache.ignite.internal.processors.hadoop.impl.shuffle.collections.HadoopConcurrentHashMultimapSelftest;
import org.apache.ignite.internal.processors.hadoop.impl.shuffle.collections.HadoopHashMapSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.shuffle.collections.HadoopSkipListSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.shuffle.streams.HadoopDataStreamSelfTest;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import static org.apache.ignite.testframework.GridTestUtils.modeToPermissionSet;
/**
* Test suite for Hadoop Map Reduce engine.
*/
public class IgniteHadoopTestSuite extends TestSuite {
/**
* @return Test suite.
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite() throws Exception {
downloadHadoop();
downloadHive();
final ClassLoader ldr = new HadoopTestClassLoader();
TestSuite suite = new TestSuite("Ignite Hadoop MR Test Suite");
suite.addTest(new TestSuite(ldr.loadClass(HadoopUserLibsSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopWeightedMapReducePlannerTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(BasicUserNameMapperSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(KerberosUserNameMapperSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(ChainedUserNameMapperSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(KerberosHadoopFileSystemFactorySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopTeraSortTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyFullMapReduceTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfs20FileSystemLoopbackPrimarySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfsDualSyncSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfsDualAsyncSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(Hadoop1OverIgfsDualSyncTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(Hadoop1OverIgfsDualAsyncTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(Hadoop1OverIgfsProxyTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopFIleSystemFactorySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemClientBasedPrimarySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemClientBasedDualAsyncSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemClientBasedDualSyncSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemClientBasedProxySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalToClientPrimarySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalToClientDualSyncSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalToClientDualAsyncSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalToClientProxySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemClientSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoggerStateSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoggerSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemHandshakeSelfTest.class.getName())));
suite.addTest(IgfsEventsTestSuite.suiteNoarchOnly());
suite.addTest(new TestSuite(ldr.loadClass(HadoopFileSystemsTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopValidationSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopJobTrackerSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopHashMapSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopDataStreamSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopConcurrentHashMultimapSelftest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopSkipListSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopTaskExecutionSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopV2JobSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopSerializationWrapperSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopSplitWrapperSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopTasksV1Test.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopTasksV2Test.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopMapReduceTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopWeightedPlannerMapReduceTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopNoHadoopMapReduceTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopMapReduceErrorResilienceTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopMapReduceEmbeddedSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopSortingTest.class.getName())));
// suite.addTest(new TestSuite(ldr.loadClass(HadoopExternalTaskExecutionSelfTest.class.getName())));
// suite.addTest(new TestSuite(ldr.loadClass(HadoopExternalCommunicationSelfTest.class.getName())));
// suite.addTest(new TestSuite(ldr.loadClass(HadoopSortingExternalTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopGroupingTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopClientProtocolSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopClientProtocolEmbeddedSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopClientProtocolMultipleServersSelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopCommandLineTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopSecondaryFileSystemConfigurationTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopTxConfigCacheTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemClientBasedOpenTest.class.getName())));
return suite;
}
/**
* @throws Exception If failed.
*/
public static void downloadHive() throws Exception {
String ver = IgniteSystemProperties.getString("hive.version", "1.2.1");
X.println("Will use Hive version: " + ver);
String downloadPath = "hive/hive-" + ver + "/apache-hive-" + ver + "-bin.tar.gz";
download("Hive", "HIVE_HOME", downloadPath, "apache-hive-" + ver + "-bin");
}
/**
* @throws Exception If failed.
*/
public static void downloadHadoop() throws Exception {
String ver = IgniteSystemProperties.getString("hadoop.version", "2.4.1");
X.println("Will use Hadoop version: " + ver);
String downloadPath = "hadoop/core/hadoop-" + ver + "/hadoop-" + ver + ".tar.gz";
download("Hadoop", "HADOOP_HOME", downloadPath, "hadoop-" + ver);
}
/**
* Downloads and extracts an Apache product.
*
* @param appName Name of application for log messages.
* @param homeVariable Pointer to home directory of the component.
* @param downloadPath Relative download path of tar package.
* @param destName Local directory name to install component.
* @throws Exception If failed.
*/
private static void download(String appName, String homeVariable, String downloadPath, String destName)
throws Exception {
String homeVal = IgniteSystemProperties.getString(homeVariable);
if (!F.isEmpty(homeVal) && new File(homeVal).isDirectory()) {
X.println(homeVariable + " is set to: " + homeVal);
return;
}
List<String> urls = F.asList(
"http://archive.apache.org/dist/",
"http://apache-mirror.rbc.ru/pub/apache/",
"http://www.eu.apache.org/dist/",
"http://www.us.apache.org/dist/");
String tmpPath = System.getProperty("java.io.tmpdir");
X.println("tmp: " + tmpPath);
final File install = new File(tmpPath + File.separatorChar + "__hadoop");
final File home = new File(install, destName);
X.println("Setting " + homeVariable + " to " + home.getAbsolutePath());
System.setProperty(homeVariable, home.getAbsolutePath());
final File successFile = new File(home, "__success");
if (home.exists()) {
if (successFile.exists()) {
X.println(appName + " distribution already exists.");
return;
}
X.println(appName + " distribution is invalid and it will be deleted.");
if (!U.delete(home))
throw new IOException("Failed to delete directory: " + home.getAbsolutePath());
}
for (String url : urls) {
if (!(install.exists() || install.mkdirs()))
throw new IOException("Failed to create directory: " + install.getAbsolutePath());
URL u = new URL(url + downloadPath);
X.println("Attempting to download from: " + u);
try {
URLConnection c = u.openConnection();
c.connect();
try (TarArchiveInputStream in = new TarArchiveInputStream(new GzipCompressorInputStream(
new BufferedInputStream(c.getInputStream(), 32 * 1024)))) {
TarArchiveEntry entry;
while ((entry = in.getNextTarEntry()) != null) {
File dest = new File(install, entry.getName());
if (entry.isDirectory()) {
if (!dest.mkdirs())
throw new IllegalStateException();
}
else if (entry.isSymbolicLink()) {
// Important: in Hadoop installation there are symlinks, we need to create them:
Path theLinkItself = Paths.get(install.getAbsolutePath(), entry.getName());
Path linkTarget = Paths.get(entry.getLinkName());
Files.createSymbolicLink(theLinkItself, linkTarget);
}
else {
File parent = dest.getParentFile();
if (!(parent.exists() || parent.mkdirs()))
throw new IllegalStateException();
X.print(" [" + dest);
try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(dest, false),
128 * 1024)) {
U.copy(in, out);
out.flush();
}
Files.setPosixFilePermissions(dest.toPath(), modeToPermissionSet(entry.getMode()));
X.println("]");
}
}
}
if (successFile.createNewFile())
return;
}
catch (Exception e) {
e.printStackTrace();
U.delete(home);
}
}
throw new IllegalStateException("Failed to install " + appName + ".");
}
}