package com.github.sakserv.minicluster.impl; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.Properties; import com.github.sakserv.minicluster.oozie.sharelib.Framework; import com.github.sakserv.minicluster.oozie.sharelib.util.OozieShareLibUtil; import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.oozie.client.CoordinatorJob; import org.apache.oozie.client.Job; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.WorkflowJob; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.github.sakserv.minicluster.config.ConfigVars; import com.github.sakserv.propertyparser.PropertyParser; public class OozieLocalServerIntegrationTest { // Logger private static final Logger LOG = LoggerFactory.getLogger(OozieLocalServerIntegrationTest.class); // Test path private static final String TEST_INPUT_DIR = "/tmp/test_input_dir"; private static final String TEST_INPUT_FILE = "test_input.txt"; private static final String TEST_OUTPUT_DIR = "/tmp/test_output_dir"; // Setup the property parser private static PropertyParser propertyParser; private static OozieLocalServer oozieLocalServer; private static HdfsLocalCluster hdfsLocalCluster; private static MRLocalCluster mrLocalCluster; static { try { propertyParser = new PropertyParser(ConfigVars.DEFAULT_PROPS_FILE); propertyParser.parsePropsFile(); } catch (IOException e) { LOG.error("Unable to load property file: {}", propertyParser.getProperty(ConfigVars.DEFAULT_PROPS_FILE)); } } @BeforeClass public static void setUp() throws Exception { hdfsLocalCluster = new HdfsLocalCluster.Builder() .setHdfsNamenodePort(Integer.parseInt(propertyParser.getProperty(ConfigVars.HDFS_NAMENODE_PORT_KEY))) .setHdfsTempDir(propertyParser.getProperty(ConfigVars.HDFS_TEMP_DIR_KEY)) .setHdfsNumDatanodes(Integer.parseInt(propertyParser.getProperty(ConfigVars.HDFS_NUM_DATANODES_KEY))) .setHdfsEnablePermissions( Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.HDFS_ENABLE_PERMISSIONS_KEY))) .setHdfsFormat(Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.HDFS_FORMAT_KEY))) .setHdfsEnableRunningUserAsProxyUser(Boolean.parseBoolean( propertyParser.getProperty(ConfigVars.HDFS_ENABLE_RUNNING_USER_AS_PROXY_USER))) .setHdfsConfig(new Configuration()) .build(); hdfsLocalCluster.start(); mrLocalCluster = new MRLocalCluster.Builder() .setNumNodeManagers(Integer.parseInt(propertyParser.getProperty(ConfigVars.YARN_NUM_NODE_MANAGERS_KEY))) .setJobHistoryAddress(propertyParser.getProperty(ConfigVars.MR_JOB_HISTORY_ADDRESS_KEY)) .setResourceManagerAddress(propertyParser.getProperty(ConfigVars.YARN_RESOURCE_MANAGER_ADDRESS_KEY)) .setResourceManagerHostname(propertyParser.getProperty(ConfigVars.YARN_RESOURCE_MANAGER_HOSTNAME_KEY)) .setResourceManagerSchedulerAddress(propertyParser.getProperty( ConfigVars.YARN_RESOURCE_MANAGER_SCHEDULER_ADDRESS_KEY)) .setResourceManagerResourceTrackerAddress(propertyParser.getProperty( ConfigVars.YARN_RESOURCE_MANAGER_RESOURCE_TRACKER_ADDRESS_KEY)) .setResourceManagerWebappAddress(propertyParser.getProperty( ConfigVars.YARN_RESOURCE_MANAGER_WEBAPP_ADDRESS_KEY)) .setUseInJvmContainerExecutor(Boolean.parseBoolean(propertyParser.getProperty( ConfigVars.YARN_USE_IN_JVM_CONTAINER_EXECUTOR_KEY))) .setHdfsDefaultFs(hdfsLocalCluster.getHdfsConfig().get("fs.defaultFS")) .setConfig(hdfsLocalCluster.getHdfsConfig()) .build(); mrLocalCluster.start(); oozieLocalServer = new OozieLocalServer.Builder() .setOozieTestDir(propertyParser.getProperty(ConfigVars.OOZIE_TEST_DIR_KEY)) .setOozieHomeDir(propertyParser.getProperty(ConfigVars.OOZIE_HOME_DIR_KEY)) .setOozieUsername(System.getProperty("user.name")) .setOozieGroupname(propertyParser.getProperty(ConfigVars.OOZIE_GROUPNAME_KEY)) .setOozieYarnResourceManagerAddress(propertyParser.getProperty( ConfigVars.YARN_RESOURCE_MANAGER_ADDRESS_KEY)) .setOozieHdfsDefaultFs(hdfsLocalCluster.getHdfsConfig().get("fs.defaultFS")) .setOozieConf(mrLocalCluster.getConfig()) .setOozieHdfsShareLibDir(propertyParser.getProperty(ConfigVars.OOZIE_HDFS_SHARE_LIB_DIR_KEY)) .setOozieShareLibCreate(Boolean.parseBoolean( propertyParser.getProperty(ConfigVars.OOZIE_SHARE_LIB_CREATE_KEY))) .setOozieLocalShareLibCacheDir(propertyParser.getProperty( ConfigVars.OOZIE_LOCAL_SHARE_LIB_CACHE_DIR_KEY)) .setOoziePurgeLocalShareLibCache(Boolean.parseBoolean(propertyParser.getProperty( ConfigVars.OOZIE_PURGE_LOCAL_SHARE_LIB_CACHE_KEY))) .setOozieShareLibFrameworks(Lists.newArrayList(Framework.MAPREDUCE_STREAMING, Framework.OOZIE)) .build(); OozieShareLibUtil oozieShareLibUtil = new OozieShareLibUtil(oozieLocalServer.getOozieHdfsShareLibDir(), oozieLocalServer.getOozieShareLibCreate(), oozieLocalServer.getOozieLocalShareLibCacheDir(), oozieLocalServer.getOoziePurgeLocalShareLibCache(), hdfsLocalCluster.getHdfsFileSystemHandle(), oozieLocalServer.getOozieShareLibFrameworks()); oozieShareLibUtil.createShareLib(); oozieLocalServer.start(); } @AfterClass public static void tearDown() throws Exception { oozieLocalServer.stop(); mrLocalCluster.stop(); hdfsLocalCluster.stop(); } @Test public void testSubmitWorkflow() throws Exception { LOG.info("OOZIE: Test Submit Workflow Start"); FileSystem hdfsFs = hdfsLocalCluster.getHdfsFileSystemHandle(); OozieClient oozie = oozieLocalServer.getOozieClient(); Path appPath = new Path(hdfsFs.getHomeDirectory(), "testApp"); hdfsFs.mkdirs(new Path(appPath, "lib")); Path workflow = new Path(appPath, "workflow.xml"); // Setup input directory and file hdfsFs.mkdirs(new Path(TEST_INPUT_DIR)); hdfsFs.copyFromLocalFile( new Path(getClass().getClassLoader().getResource(TEST_INPUT_FILE).toURI()), new Path(TEST_INPUT_DIR)); //write workflow.xml String wfApp = "<workflow-app name=\"sugar-option-decision\" xmlns=\"uri:oozie:workflow:0.5\">\n" + " <global>\n" + " <job-tracker>${jobTracker}</job-tracker>\n" + " <name-node>${nameNode}</name-node>\n" + " <configuration>\n" + " <property>\n" + " <name>mapreduce.output.fileoutputformat.outputdir</name>\n" + " <value>" + TEST_OUTPUT_DIR + "</value>\n" + " </property>\n" + " <property>\n" + " <name>mapreduce.input.fileinputformat.inputdir</name>\n" + " <value>" + TEST_INPUT_DIR + "</value>\n" + " </property>\n" + " </configuration>\n" + " </global>\n" + " <start to=\"first\"/>\n" + " <action name=\"first\">\n" + " <map-reduce> <prepare><delete path=\"" + TEST_OUTPUT_DIR + "\"/></prepare></map-reduce>\n" + " <ok to=\"decision-second-option\"/>\n" + " <error to=\"kill\"/>\n" + " </action>\n" + " <decision name=\"decision-second-option\">\n" + " <switch>\n" + " <case to=\"option\">${doOption}</case>\n" + " <default to=\"second\"/>\n" + " </switch>\n" + " </decision>\n" + " <action name=\"option\">\n" + " <map-reduce> <prepare><delete path=\"" + TEST_OUTPUT_DIR + "\"/></prepare></map-reduce>\n" + " <ok to=\"second\"/>\n" + " <error to=\"kill\"/>\n" + " </action>\n" + " <action name=\"second\">\n" + " <map-reduce> <prepare><delete path=\"" + TEST_OUTPUT_DIR + "\"/></prepare></map-reduce>\n" + " <ok to=\"end\"/>\n" + " <error to=\"kill\"/>\n" + " </action>\n" + " <kill name=\"kill\">\n" + " <message>\n" + " Failed to workflow, error message[${wf: errorMessage (wf: lastErrorNode ())}]\n" + " </message>\n" + " </kill>\n" + " <end name=\"end\"/>\n" + "</workflow-app>"; Writer writer = new OutputStreamWriter(hdfsFs.create(workflow)); writer.write(wfApp); writer.close(); //write job.properties Properties conf = oozie.createConfiguration(); conf.setProperty(OozieClient.APP_PATH, workflow.toString()); conf.setProperty(OozieClient.USER_NAME, UserGroupInformation.getCurrentUser().getUserName()); conf.setProperty("nameNode", "hdfs://localhost:" + hdfsLocalCluster.getHdfsNamenodePort()); conf.setProperty("jobTracker", mrLocalCluster.getResourceManagerAddress()); conf.setProperty("doOption", "true"); //submit and check final String jobId = oozie.run(conf); WorkflowJob wf = oozie.getJobInfo(jobId); assertNotNull(wf); assertEquals(WorkflowJob.Status.RUNNING, wf.getStatus()); while(true){ Thread.sleep(1000); wf = oozie.getJobInfo(jobId); if(wf.getStatus() == WorkflowJob.Status.FAILED || wf.getStatus() == WorkflowJob.Status.KILLED || wf.getStatus() == WorkflowJob.Status.PREP || wf.getStatus() == WorkflowJob.Status.SUCCEEDED){ break; } } wf = oozie.getJobInfo(jobId); assertEquals(WorkflowJob.Status.SUCCEEDED, wf.getStatus()); LOG.info("OOZIE: Workflow: {}", wf.toString()); hdfsFs.close(); } @Test public void testSubmitCoordinator() throws Exception { LOG.info("OOZIE: Test Submit Coordinator Start"); FileSystem hdfsFs = hdfsLocalCluster.getHdfsFileSystemHandle(); OozieClient oozie = oozieLocalServer.getOozieCoordClient(); Path appPath = new Path(hdfsFs.getHomeDirectory(), "testApp"); hdfsFs.mkdirs(new Path(appPath, "lib")); Path workflow = new Path(appPath, "workflow.xml"); Path coordinator = new Path(appPath, "coordinator.xml"); //write workflow.xml String wfApp = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='test-wf'>" + " <start to='end'/>" + " <end name='end'/>" + "</workflow-app>"; String coordApp = "<coordinator-app timezone='UTC' end='2016-07-26T02:26Z' start='2016-07-26T01:26Z' frequency='${coord:hours(1)}' name='test-coordinator' xmlns='uri:oozie:coordinator:0.4'>" + " <action>" + " <workflow>" + " <app-path>" + workflow.toString() + "</app-path>" + " </workflow>" + " </action>" + "</coordinator-app>"; Writer writer = new OutputStreamWriter(hdfsFs.create(workflow)); writer.write(wfApp); writer.close(); Writer coordWriter = new OutputStreamWriter(hdfsFs.create(coordinator)); coordWriter.write(coordApp); coordWriter.close(); //write job.properties Properties conf = oozie.createConfiguration(); conf.setProperty(OozieClient.COORDINATOR_APP_PATH, coordinator.toString()); conf.setProperty(OozieClient.USER_NAME, UserGroupInformation.getCurrentUser().getUserName()); //submit and check final String jobId = oozie.submit(conf); CoordinatorJob coord = oozie.getCoordJobInfo(jobId); assertNotNull(coord); assertEquals(Job.Status.PREP, coord.getStatus()); LOG.info("OOZIE: Coordinator: {}", coord.toString()); hdfsFs.close(); } @Test public void testOozieShareLib() throws Exception { LOG.info("OOZIE: Test Oozie Share Lib Start"); FileSystem hdfsFs = hdfsLocalCluster.getHdfsFileSystemHandle(); OozieShareLibUtil oozieShareLibUtil = new OozieShareLibUtil(oozieLocalServer.getOozieHdfsShareLibDir(), oozieLocalServer.getOozieShareLibCreate(), oozieLocalServer.getOozieLocalShareLibCacheDir(), oozieLocalServer.getOoziePurgeLocalShareLibCache(), hdfsFs, oozieLocalServer.getOozieShareLibFrameworks()); oozieShareLibUtil.createShareLib(); // Validate the share lib dir was created and contains a single directory FileStatus[] fileStatuses = hdfsFs.listStatus(new Path(oozieLocalServer.getOozieHdfsShareLibDir())); assertEquals(1, fileStatuses.length); hdfsFs.close(); } }