/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.falcon.regression;
import org.apache.falcon.regression.Entities.FeedMerlin;
import org.apache.falcon.regression.core.bundle.Bundle;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.feed.ActionType;
import org.apache.falcon.entity.v0.feed.ClusterType;
import org.apache.falcon.regression.core.helpers.ColoHelper;
import org.apache.falcon.regression.core.util.BundleUtil;
import org.apache.falcon.regression.core.util.HadoopUtil;
import org.apache.falcon.regression.core.util.InstanceUtil;
import org.apache.falcon.regression.core.util.OSUtil;
import org.apache.falcon.regression.core.util.TimeUtil;
import org.apache.falcon.regression.core.util.Util;
import org.apache.falcon.regression.testHelper.BaseTestClass;
import org.apache.falcon.resource.InstancesSummaryResult;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.oozie.client.CoordinatorAction.Status;
import org.apache.oozie.client.OozieClientException;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import javax.xml.bind.JAXBException;
import java.io.IOException;
import java.net.URISyntaxException;
import java.text.ParseException;
import java.util.List;
/** This test currently provide minimum verification. More detailed test should be added:
1. process : test summary single cluster few instance some future some past
2. process : test multiple cluster, full past on one cluster, full future on one cluster,
half future / past on third one
3. feed : same as test 1 for feed
4. feed : same as test 2 for feed
*/
@Test(groups = { "distributed", "embedded" })
public class InstanceSummaryTest extends BaseTestClass {
private String baseTestHDFSDir = cleanAndGetTestDir();
private String feedInputPath = baseTestHDFSDir + "/testInputData" + MINUTE_DATE_PATTERN;
private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator";
private String startTime;
private String endTime;
private ColoHelper cluster3 = servers.get(2);
private Bundle processBundle;
private String processName;
@BeforeClass(alwaysRun = true)
public void createTestData() throws Exception {
uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
startTime = TimeUtil.get20roundedTime(TimeUtil.getTimeWrtSystemTime(-20));
endTime = TimeUtil.getTimeWrtSystemTime(60);
String startTimeData = TimeUtil.addMinsToTime(startTime, -100);
List<String> dataDates = TimeUtil.getMinuteDatesOnEitherSide(startTimeData, endTime, 20);
for (FileSystem fs : serverFS) {
HadoopUtil.deleteDirIfExists(Util.getPathPrefix(feedInputPath), fs);
HadoopUtil.flattenAndPutDataInFolder(fs, OSUtil.NORMAL_INPUT,
Util.getPathPrefix(feedInputPath), dataDates);
}
}
@BeforeMethod(alwaysRun = true)
public void setup() throws Exception {
processBundle = new Bundle(BundleUtil.readELBundle(), cluster3);
processBundle.generateUniqueBundle(this);
processBundle.setInputFeedDataPath(feedInputPath);
processBundle.setOutputFeedLocationData(baseTestHDFSDir + "/output" + MINUTE_DATE_PATTERN);
processBundle.setProcessWorkflow(aggregateWorkflowDir);
for (int i = 0; i < 3; i++) {
bundles[i] = new Bundle(BundleUtil.readELBundle(), servers.get(i));
bundles[i].generateUniqueBundle(this);
bundles[i].setProcessWorkflow(aggregateWorkflowDir);
}
processName = Util.readEntityName(processBundle.getProcessData());
}
/**
* Schedule single-cluster process. Get its instances summary.
*/
@Test(enabled = true, timeOut = 1200000)
public void testSummarySingleClusterProcess()
throws URISyntaxException, JAXBException, IOException, ParseException,
OozieClientException, AuthenticationException, InterruptedException {
processBundle.setProcessValidity(startTime, endTime);
processBundle.submitFeedsScheduleProcess(prism);
InstanceUtil.waitTillInstancesAreCreated(serverOC.get(2), processBundle.getProcessData(), 0);
// start only at start time
InstancesSummaryResult r = prism.getProcessHelper()
.getInstanceSummary(processName, "?start=" + startTime);
InstanceUtil.waitTillInstanceReachState(serverOC.get(2), processName, 2,
Status.SUCCEEDED, EntityType.PROCESS);
//AssertUtil.assertSucceeded(r);
//start only before process start
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + TimeUtil.addMinsToTime(startTime, -100));
//AssertUtil.assertFailed(r,"response should have failed");
//start only after process end
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + TimeUtil.addMinsToTime(startTime, 120));
//start only at mid specific instance
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + TimeUtil.addMinsToTime(startTime, 10));
//start only in between 2 instance
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + TimeUtil.addMinsToTime(startTime, 7));
//start and end at start and end
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + startTime + "&end=" + endTime);
//start in between and end at end
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + TimeUtil.addMinsToTime(startTime, 14) + "&end=" + endTime);
//start at start and end between
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + startTime + "&end=" + TimeUtil.addMinsToTime(endTime, -20));
// start and end in between
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + TimeUtil.addMinsToTime(startTime, 20)
+ "&end=" + TimeUtil.addMinsToTime(endTime, -13));
//start before start with end in between
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + TimeUtil.addMinsToTime(startTime, -100)
+ "&end=" + TimeUtil.addMinsToTime(endTime, -37));
//start in between and end after end
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + TimeUtil.addMinsToTime(startTime, 60)
+ "&end=" + TimeUtil.addMinsToTime(endTime, 100));
// both start end out od range
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + TimeUtil.addMinsToTime(startTime, -100)
+ "&end=" + TimeUtil.addMinsToTime(endTime, 100));
// end only
r = prism.getProcessHelper().getInstanceSummary(processName,
"?end=" + TimeUtil.addMinsToTime(endTime, -30));
}
/**
* Adjust multi-cluster process. Submit and schedule it. Get its instances summary.
*/
@Test(enabled = true, timeOut = 1200000, groups = "multiCluster")
public void testSummaryMultiClusterProcess() throws JAXBException,
ParseException, IOException, URISyntaxException, AuthenticationException,
InterruptedException {
processBundle.setProcessValidity(startTime, endTime);
processBundle.addClusterToBundle(bundles[1].getClusters().get(0),
ClusterType.SOURCE, null, null);
processBundle.addClusterToBundle(bundles[2].getClusters().get(0),
ClusterType.SOURCE, null, null);
processBundle.submitFeedsScheduleProcess(prism);
InstancesSummaryResult r = prism.getProcessHelper()
.getInstanceSummary(processName, "?start=" + startTime);
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + startTime + "&end=" + endTime);
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + startTime + "&end=" + endTime);
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + startTime + "&end=" + endTime);
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + startTime + "&end=" + endTime);
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + startTime + "&end=" + endTime);
r = prism.getProcessHelper().getInstanceSummary(processName,
"?start=" + startTime + "&end=" + endTime);
}
/**
* Adjust multi-cluster feed. Submit and schedule it. Get its instances summary.
*/
@Test(enabled = true, timeOut = 1200000, groups = "multiCluster")
public void testSummaryMultiClusterFeed() throws JAXBException, ParseException, IOException,
URISyntaxException, OozieClientException, AuthenticationException,
InterruptedException {
//create desired feed
String feed = bundles[0].getDataSets().get(0);
//cluster_1 is target, cluster_2 is source and cluster_3 is neutral
feed = FeedMerlin.fromString(feed).clearFeedClusters().toString();
feed = FeedMerlin.fromString(feed).addFeedCluster(
new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0)))
.withRetention("days(100000)", ActionType.DELETE)
.withValidity(startTime, "2099-10-01T12:10Z")
.build()).toString();
feed = FeedMerlin.fromString(feed).addFeedCluster(
new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0)))
.withRetention("days(100000)", ActionType.DELETE)
.withValidity(startTime, "2099-10-01T12:25Z")
.withClusterType(ClusterType.TARGET)
.withDataLocation(feedInputPath)
.build()).toString();
feed = FeedMerlin.fromString(feed).addFeedCluster(
new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0)))
.withRetention("days(100000)", ActionType.DELETE)
.withValidity(startTime, "2099-01-01T00:00Z")
.withClusterType(ClusterType.SOURCE)
.withDataLocation(feedInputPath)
.build()).toString();
//submit clusters
Bundle.submitCluster(bundles[0], bundles[1], bundles[2]);
//create test data on cluster_2
/*InstanceUtil.createDataWithinDatesAndPrefix(cluster2,
InstanceUtil.oozieDateToDate(startTime),
InstanceUtil.oozieDateToDate(InstanceUtil.getTimeWrtSystemTime(60)),
feedInputPath, 1);*/
//submit and schedule feed
prism.getFeedHelper().submitAndSchedule(feed);
InstancesSummaryResult r = prism.getFeedHelper()
.getInstanceSummary(Util.readEntityName(feed), "?start=" + startTime);
r = prism.getFeedHelper().getInstanceSummary(Util.readEntityName(feed),
"?start=" + startTime + "&end=" + TimeUtil.addMinsToTime(endTime, -20));
}
@AfterMethod(alwaysRun = true)
public void tearDown() throws IOException {
removeTestClassEntities();
}
}