/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hive.hcatalog.templeton;
import java.io.IOException;
import java.util.ArrayList;
import java.util.concurrent.TimeoutException;
import org.eclipse.jetty.http.HttpStatus;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import static org.junit.Assert.assertTrue;
/*
* Test submission of concurrent job requests with the controlled number of concurrent
* Requests and job request execution time outs. Verify that we get appropriate exceptions
* and exception message.
*/
public class TestConcurrentJobRequestsThreadsAndTimeout extends ConcurrentJobRequestsTestBase {
private static AppConfig config;
private static QueueStatusBean statusBean;
private static String statusTooManyRequestsExceptionMessage;
private static String listTooManyRequestsExceptionMessage;
private static String submitTooManyRequestsExceptionMessage;
@Rule
public ExpectedException exception = ExpectedException.none();
@BeforeClass
public static void setUp() {
final String[] args = new String[] {};
Main main = new Main(args);
config = main.getAppConfigInstance();
config.setInt(AppConfig.JOB_STATUS_MAX_THREADS, 5);
config.setInt(AppConfig.JOB_LIST_MAX_THREADS, 5);
config.setInt(AppConfig.JOB_SUBMIT_MAX_THREADS, 5);
config.setInt(AppConfig.JOB_SUBMIT_TIMEOUT, 5);
config.setInt(AppConfig.JOB_STATUS_TIMEOUT, 5);
config.setInt(AppConfig.JOB_LIST_TIMEOUT, 5);
config.setInt(AppConfig.JOB_TIMEOUT_TASK_RETRY_COUNT, 4);
config.setInt(AppConfig.JOB_TIMEOUT_TASK_RETRY_INTERVAL, 1);
statusBean = new QueueStatusBean("job_1000", "Job not found");
statusTooManyRequestsExceptionMessage = "Unable to service the status job request as "
+ "templeton service is busy with too many status job requests. "
+ "Please wait for some time before retrying the operation. "
+ "Please refer to the config templeton.parallellism.job.status "
+ "to configure concurrent requests.";
listTooManyRequestsExceptionMessage = "Unable to service the list job request as "
+ "templeton service is busy with too many list job requests. "
+ "Please wait for some time before retrying the operation. "
+ "Please refer to the config templeton.parallellism.job.list "
+ "to configure concurrent requests.";
submitTooManyRequestsExceptionMessage = "Unable to service the submit job request as "
+ "templeton service is busy with too many submit job requests. "
+ "Please wait for some time before retrying the operation. "
+ "Please refer to the config templeton.parallellism.job.submit "
+ "to configure concurrent requests.";
}
@Test
public void ConcurrentJobsStatusTooManyRequestsException() {
try {
JobRunnable jobRunnable = ConcurrentJobsStatus(6, config, false, false,
statusJobHelper.getDelayedResonseAnswer(4, statusBean));
verifyTooManyRequestsException(jobRunnable.exception, this.statusTooManyRequestsExceptionMessage);
} catch (Exception e) {
assertTrue(false);
}
}
@Test
public void ConcurrentListJobsTooManyRequestsException() {
try {
JobRunnable jobRunnable = ConcurrentListJobs(6, config, false, false,
listJobHelper.getDelayedResonseAnswer(4, new ArrayList<JobItemBean>()));
verifyTooManyRequestsException(jobRunnable.exception, this.listTooManyRequestsExceptionMessage);
} catch (Exception e) {
assertTrue(false);
}
}
@Test
public void ConcurrentSubmitJobsTooManyRequestsException() {
try {
JobRunnable jobRunnable = SubmitConcurrentJobs(6, config, false, false,
submitJobHelper.getDelayedResonseAnswer(4, 0),
killJobHelper.getDelayedResonseAnswer(0, statusBean), "job_1000");
verifyTooManyRequestsException(jobRunnable.exception, this.submitTooManyRequestsExceptionMessage);
} catch (Exception e) {
assertTrue(false);
}
}
@Test
public void ConcurrentJobsStatusTimeOutException() {
try {
JobRunnable jobRunnable = ConcurrentJobsStatus(5, config, false, false,
statusJobHelper.getDelayedResonseAnswer(6, statusBean));
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof TimeoutException);
String expectedMessage = "Status job request got timed out. Please wait for some time before "
+ "retrying the operation. Please refer to the config "
+ "templeton.job.status.timeout to configure job request time out.";
assertTrue(jobRunnable.exception.getMessage().contains(expectedMessage));
/*
* Verify that new job requests should succeed with no issues.
*/
jobRunnable = ConcurrentJobsStatus(5, config, false, false,
statusJobHelper.getDelayedResonseAnswer(0, statusBean));
assertTrue(jobRunnable.exception == null);
} catch (Exception e) {
assertTrue(false);
}
}
@Test
public void ConcurrentListJobsTimeOutException() {
try {
JobRunnable jobRunnable = ConcurrentListJobs(5, config, false, false,
listJobHelper.getDelayedResonseAnswer(6, new ArrayList<JobItemBean>()));
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof TimeoutException);
String expectedMessage = "List job request got timed out. Please wait for some time before "
+ "retrying the operation. Please refer to the config "
+ "templeton.job.list.timeout to configure job request time out.";
assertTrue(jobRunnable.exception.getMessage().contains(expectedMessage));
/*
* Verify that new job requests should succeed with no issues.
*/
jobRunnable = ConcurrentListJobs(5, config, false, false,
listJobHelper.getDelayedResonseAnswer(1, new ArrayList<JobItemBean>()));
assertTrue(jobRunnable.exception == null);
} catch (Exception e) {
assertTrue(false);
}
}
@Test
public void ConcurrentSubmitJobsTimeOutException() {
try {
JobRunnable jobRunnable = SubmitConcurrentJobs(5, config, false, false,
submitJobHelper.getDelayedResonseAnswer(6, 0),
killJobHelper.getDelayedResonseAnswer(0, statusBean), "job_1000");
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof QueueException);
String expectedMessage = "Submit job request got timed out. Please wait for some time before "
+ "retrying the operation. Please refer to the config "
+ "templeton.job.submit.timeout to configure job request time out.";
assertTrue(jobRunnable.exception.getMessage().contains(expectedMessage));
/*
* For submit operation, tasks are not cancelled. Verify that new job request
* should fail with TooManyRequestsException.
*/
jobRunnable = SubmitConcurrentJobs(1, config, false, false,
submitJobHelper.getDelayedResonseAnswer(0, 0),
killJobHelper.getDelayedResonseAnswer(0, statusBean), "job_1000");
verifyTooManyRequestsException(jobRunnable.exception, this.submitTooManyRequestsExceptionMessage);
/*
* Sleep until all threads with clean up tasks are completed.
*/
Thread.sleep(2000);
/*
* Now, tasks would have passed. Verify that new job requests should succeed with no issues.
*/
jobRunnable = SubmitConcurrentJobs(5, config, false, false,
submitJobHelper.getDelayedResonseAnswer(0, 0),
killJobHelper.getDelayedResonseAnswer(0, statusBean), "job_1000");
assertTrue(jobRunnable.exception == null);
} catch (Exception e) {
assertTrue(false);
}
}
@Test
public void ConcurrentStatusJobsVerifyExceptions() {
try {
/*
* Trigger kill threads and verify we get InterruptedException and expected Message.
*/
int timeoutTaskDelay = 4;
JobRunnable jobRunnable = ConcurrentJobsStatus(5, config, true, false,
statusJobHelper.getDelayedResonseAnswer(timeoutTaskDelay, statusBean));
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof InterruptedException);
String expectedMessage = "Status job request got interrupted. Please wait for some time before "
+ "retrying the operation.";
assertTrue(jobRunnable.exception.getMessage().contains(expectedMessage));
/*
* Interrupt all thread and verify we get InterruptedException and expected Message.
*/
jobRunnable = ConcurrentJobsStatus(5, config, false, true,
statusJobHelper.getDelayedResonseAnswer(timeoutTaskDelay, statusBean));
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof InterruptedException);
assertTrue(jobRunnable.exception.getMessage().contains(expectedMessage));
/*
* Raise custom exception like IOException and verify expected Message.
*/
jobRunnable = ConcurrentJobsStatus(5, config, false, false,
statusJobHelper.getIOExceptionAnswer());
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception.getCause() instanceof IOException);
/*
* Now new job requests should succeed as status operation has no cancel threads.
*/
jobRunnable = ConcurrentJobsStatus(5, config, false, false,
statusJobHelper.getDelayedResonseAnswer(0, statusBean));
assertTrue(jobRunnable.exception == null);
} catch (Exception e) {
assertTrue(false);
}
}
@Test
public void ConcurrentListJobsVerifyExceptions() {
try {
/*
* Trigger kill threads and verify we get InterruptedException and expected Message.
*/
int timeoutTaskDelay = 4;
JobRunnable jobRunnable = ConcurrentListJobs(5, config, true, false,
listJobHelper.getDelayedResonseAnswer(timeoutTaskDelay, new ArrayList<JobItemBean>()));
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof InterruptedException);
String expectedMessage = "List job request got interrupted. Please wait for some time before "
+ "retrying the operation.";
assertTrue(jobRunnable.exception.getMessage().contains(expectedMessage));
/*
* Interrupt all thread and verify we get InterruptedException and expected Message.
*/
jobRunnable = ConcurrentListJobs(5, config, false, true,
listJobHelper.getDelayedResonseAnswer(timeoutTaskDelay, new ArrayList<JobItemBean>()));
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof InterruptedException);
assertTrue(jobRunnable.exception.getMessage().contains(expectedMessage));
/*
* Raise custom exception like IOException and verify expected Message.
*/
jobRunnable = ConcurrentListJobs(5, config, false, false,
listJobHelper.getIOExceptionAnswer());
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception.getCause() instanceof IOException);
/*
* Now new job requests should succeed as list operation has no cancel threads.
*/
jobRunnable = ConcurrentListJobs(5, config, false, false,
listJobHelper.getDelayedResonseAnswer(0, new ArrayList<JobItemBean>()));
assertTrue(jobRunnable.exception == null);
} catch (Exception e) {
assertTrue(false);
}
}
@Test
public void ConcurrentSubmitJobsVerifyExceptions() {
try {
int timeoutTaskDelay = 4;
/*
* Raise custom exception like IOException and verify expected Message.
* This should not invoke cancel operation.
*/
JobRunnable jobRunnable = SubmitConcurrentJobs(1, config, false, false,
submitJobHelper.getIOExceptionAnswer(),
killJobHelper.getDelayedResonseAnswer(timeoutTaskDelay, statusBean), "job_1002");
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof QueueException);
assertTrue(jobRunnable.exception.getMessage().contains("IOException raised manually."));
/*
* Raise custom exception like IOException and verify expected Message.
* This should not invoke cancel operation.
*/
jobRunnable = SubmitConcurrentJobs(1, config, false, false,
submitJobHelper.getOutOfMemoryErrorAnswer(),
killJobHelper.getDelayedResonseAnswer(timeoutTaskDelay, statusBean), "job_1003");
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof QueueException);
assertTrue(jobRunnable.exception.getMessage().contains("OutOfMemoryError raised manually."));
/*
* Trigger kill threads and verify that we get InterruptedException and expected
* Message. This should raise 3 kill operations and ensure that retries keep the time out
* occupied for 4 sec.
*/
jobRunnable = SubmitConcurrentJobs(3, config, true, false,
submitJobHelper.getDelayedResonseAnswer(2, 0),
killJobHelper.getDelayedResonseAnswer(timeoutTaskDelay, statusBean), "job_1000");
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof QueueException);
String expectedMessage = "Submit job request got interrupted. Please wait for some time "
+ "before retrying the operation.";
assertTrue(jobRunnable.exception.getMessage().contains(expectedMessage));
/*
* Interrupt all threads and verify we get InterruptedException and expected
* Message. Also raise 2 kill operations and ensure that retries keep the time out
* occupied for 4 sec.
*/
jobRunnable = SubmitConcurrentJobs(2, config, false, true,
submitJobHelper.getDelayedResonseAnswer(2, 0),
killJobHelper.getDelayedResonseAnswer(0, statusBean), "job_1001");
assertTrue(jobRunnable.exception != null);
assertTrue(jobRunnable.exception instanceof QueueException);
assertTrue(jobRunnable.exception.getMessage().contains(expectedMessage));
/*
* For submit operation, tasks are not cancelled. Verify that new job request
* should fail with TooManyRequestsException.
*/
jobRunnable = SubmitConcurrentJobs(1, config, false, false,
submitJobHelper.getDelayedResonseAnswer(0, 0),
killJobHelper.getDelayedResonseAnswer(0, statusBean), "job_1002");
verifyTooManyRequestsException(jobRunnable.exception, this.submitTooManyRequestsExceptionMessage);
/*
* Sleep until all threads with clean up tasks are completed. 2 seconds completing task
* and 1 sec grace period.
*/
Thread.sleep((timeoutTaskDelay + 2 + 1) * 1000);
/*
* Now new job requests should succeed as all cancel threads would have completed.
*/
jobRunnable = SubmitConcurrentJobs(5, config, false, false,
submitJobHelper.getDelayedResonseAnswer(0, 0),
killJobHelper.getDelayedResonseAnswer(0, statusBean), "job_1004");
assertTrue(jobRunnable.exception == null);
} catch (Exception e) {
assertTrue(false);
}
}
private void verifyTooManyRequestsException(Throwable exception, String expectedMessage) {
assertTrue(exception != null);
assertTrue(exception instanceof TooManyRequestsException);
TooManyRequestsException ex = (TooManyRequestsException)exception;
assertTrue(ex.httpCode == TooManyRequestsException.TOO_MANY_REQUESTS_429);
assertTrue(exception.getMessage().contains(expectedMessage));
}
}