/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lens.server;
import static org.apache.lens.server.LensServerTestUtil.createTable;
import static org.apache.lens.server.LensServerTestUtil.loadData;
import static org.apache.lens.server.api.user.MockDriverQueryHook.*;
import static org.apache.lens.server.common.RestAPITestUtil.*;
import static org.testng.Assert.*;
import java.io.*;
import java.util.*;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.lens.api.*;
import org.apache.lens.api.APIResult.Status;
import org.apache.lens.api.query.*;
import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.driver.hive.TestRemoteHiveDriver;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.lens.server.api.session.SessionService;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.lens.server.common.LenServerTestException;
import org.apache.lens.server.common.LensServerTestFileUtils;
import org.apache.lens.server.common.TestResourceFile;
import org.apache.lens.server.query.QueryExecutionServiceImpl;
import org.apache.lens.server.query.TestQueryService;
import org.apache.lens.server.session.HiveSessionService;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hive.service.Service;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.testng.Assert;
import org.testng.annotations.*;
import com.google.common.base.Optional;
import lombok.extern.slf4j.Slf4j;
/**
* The Class TestServerRestart.
*/
@Test(alwaysRun = true, groups = "restart-test", dependsOnGroups = "unit-test")
@Slf4j
public class TestServerRestart extends LensAllApplicationJerseyTest {
/** The data file. */
private File dataFile;
/**
* No of valid hive drivers that can execute queries in this test class
*/
private static final int NO_OF_HIVE_DRIVERS = 2;
/*
* (non-Javadoc)
*
* @see org.glassfish.jersey.test.JerseyTest#setUp()
*/
@BeforeTest
public void setUp() throws Exception {
super.setUp();
}
@Override
public Map<String, String> getServerConfOverWrites() {
return LensUtil.getHashMap("lens.server.state.persistence.interval.millis", "1000");
}
@AfterTest
public void tearDown() throws Exception {
super.tearDown();
}
@BeforeClass
public void restartBeforeClass() throws Exception {
// restart server with test configuration for tests
restartLensServer(getServerConf());
}
@AfterClass
public void restart() throws Exception {
// restart server with normal configuration once the tests are done.
restartLensServer();
}
/** The file created. */
private boolean fileCreated;
/** The nrows. */
public static final int NROWS = 10000;
/**
* Creates the restart test data file.
*
* @throws FileNotFoundException the file not found exception
*/
private void createRestartTestDataFile() throws FileNotFoundException {
if (fileCreated) {
return;
}
dataFile = new File(TestResourceFile.TEST_DATA_FILE.getValue());
dataFile.deleteOnExit();
PrintWriter dataFileOut = new PrintWriter(dataFile);
for (int i = 0; i < NROWS; i++) {
dataFileOut.println(i);
}
dataFileOut.flush();
dataFileOut.close();
fileCreated = true;
}
/**
* Test query service.
*
* @throws InterruptedException the interrupted exception
* @throws IOException Signals that an I/O exception has occurred.
* @throws LensException the lens exception
*/
@Test
public void testQueryService() throws InterruptedException, IOException, LensException {
log.info("Server restart test");
QueryExecutionServiceImpl queryService = LensServices.get().getService(QueryExecutionService.NAME);
Assert.assertTrue(queryService.getHealthStatus().isHealthy());
LensSessionHandle lensSessionId = queryService.openSession("foo", "bar", new HashMap<String, String>());
// Create data file
createRestartTestDataFile();
// Create a test table
createTable("test_server_restart", target(), lensSessionId, defaultMT);
loadData("test_server_restart", TestResourceFile.TEST_DATA_FILE.getValue(), target(), lensSessionId, defaultMT);
log.info("Loaded data");
// test post execute op
List<QueryHandle> launchedQueries = new ArrayList<>();
final int NUM_QUERIES = 10;
boolean isQuerySubmitterPaused = false;
QueryHandle handleForMockDriverQueryHookTest = null;
for (int i = 0; i < NUM_QUERIES; i++) {
if (!isQuerySubmitterPaused && i > NUM_QUERIES / 3) {
// Kill the query submitter thread to make sure some queries stay in accepted queue
try {
queryService.pauseQuerySubmitter(true);
log.info("Stopped query submitter");
Assert.assertFalse(queryService.getHealthStatus().isHealthy());
} catch (Exception exc) {
log.error("Could not kill query submitter", exc);
}
isQuerySubmitterPaused = true;
}
final QueryHandle handle = executeAndGetHandle(target(), Optional.of(lensSessionId),
Optional.of("select COUNT(ID) from test_server_restart"), Optional.<LensConf>absent(), defaultMT);
LensQuery ctx = getLensQuery(target(), lensSessionId, handle, defaultMT);
log.info("{} submitted query {} state: {}", i, handle, ctx.getStatus().getStatus());
launchedQueries.add(handle);
if (i == (NUM_QUERIES-1)) {
//checking this only for one of the queued queries. A queued query has all the config information available in
// server memory. (Some of the information is lost after query is purged)
testMockDriverQueryHookPostDriverSelection(queryService, handle, false);
handleForMockDriverQueryHookTest = handle;
log.info("Testing query {} for MockDriverQueryHook", handleForMockDriverQueryHookTest);
}
}
// Restart the server
log.info("Restarting lens server!");
restartLensServer(getServerConf(), true);
log.info("Restarted lens server!");
queryService = LensServices.get().getService(QueryExecutionService.NAME);
Assert.assertFalse(queryService.getHealthStatus().isHealthy());
testMockDriverQueryHookPostDriverSelection(queryService, handleForMockDriverQueryHookTest, true);
queryService.pauseQuerySubmitter(false);
Assert.assertTrue(queryService.getHealthStatus().isHealthy());
// All queries should complete after server restart
for (QueryHandle handle : launchedQueries) {
log.info("Polling query {}", handle);
try {
PersistentQueryResult resultset = getLensQueryResult(target(), lensSessionId, handle, defaultMT);
List<String> rows = TestQueryService.readResultSet(resultset, handle, true);
assertEquals(rows.size(), 1);
assertEquals(rows.get(0), "" + NROWS);
log.info("Completed {}", handle);
} catch (Exception exc) {
log.error("Failed query {}", handle, exc);
Assert.fail(exc.getMessage());
}
}
log.info("End server restart test");
LensServerTestUtil.dropTable("test_server_restart", target(), lensSessionId, defaultMT);
queryService.closeSession(lensSessionId);
}
/**
* Tests whether the driver configuration updated by mock query driver hook is
* 1. updated in LensConf wherever applicable and
* 2. is persisted and available even after server startup.
*
* @param queryService
* @param handle
* @param afterRestart
*/
private void testMockDriverQueryHookPostDriverSelection(QueryExecutionServiceImpl queryService, QueryHandle handle,
boolean afterRestart){
QueryContext ctx = queryService.getQueryContext(handle);
assertNotNull(ctx, "Make sure that the query has not been purged");
assertTrue(ctx.getStatus().queued(), "Make sure query is still in QUEUED state");
LensConf lensQueryConf = queryService.getQueryContext(handle).getLensConf();
Configuration driverConf = queryService.getQueryContext(handle).getSelectedDriverConf();
assertEquals(driverConf.get(KEY_POST_SELECT), VALUE_POST_SELECT);
assertEquals(lensQueryConf.getProperty(KEY_POST_SELECT), VALUE_POST_SELECT);
if (afterRestart) {
//This will be unavailable since if was not updated in LensConf by MockDriverQueryHook
assertNull(driverConf.get(UNSAVED_KEY_POST_SELECT));
} else {
assertEquals(driverConf.get(UNSAVED_KEY_POST_SELECT), UNSAVED_VALUE_POST_SELECT);
}
assertNull(lensQueryConf.getProperty(UNSAVED_KEY_POST_SELECT));
}
/**
* Test hive server restart.
*
* @throws Exception the exception
*/
@Test
public void testHiveServerRestart() throws Exception {
QueryExecutionServiceImpl queryService = LensServices.get().getService(QueryExecutionService.NAME);
Assert.assertTrue(queryService.getHealthStatus().isHealthy());
LensSessionHandle lensSessionId = queryService.openSession("foo", "bar", new HashMap<String, String>());
// set params
setParams(lensSessionId);
// Create data file
createRestartTestDataFile();
// Add a resource to check if its added after server restart.
HiveSessionService sessionService = LensServices.get().getService(SessionService.NAME);
Assert.assertTrue(sessionService.getHealthStatus().isHealthy());
sessionService.addResource(lensSessionId, "FILE", dataFile.toURI().toString());
log.info("@@ Added resource {}", dataFile.toURI());
// Create a test table
createTable("test_hive_server_restart", target(), lensSessionId, defaultMT);
loadData("test_hive_server_restart", TestResourceFile.TEST_DATA_FILE.getValue(), target(),
lensSessionId, defaultMT);
log.info("Loaded data");
log.info("Hive Server restart test");
// test post execute op
QueryHandle handle = executeAndGetHandle(target(), Optional.of(lensSessionId),
Optional.of("select COUNT(ID) from test_hive_server_restart"), Optional.<LensConf>absent(), defaultMT);
// wait for query to move out of QUEUED state
LensQuery ctx = getLensQuery(target(), lensSessionId, handle, defaultMT);
while (ctx.getStatus().queued()) {
ctx = getLensQuery(target(), lensSessionId, handle, defaultMT);
Thread.sleep(1000);
}
List<LensSessionImpl.ResourceEntry> sessionResources = queryService.getSession(lensSessionId)
.getLensSessionPersistInfo().getResources();
int[] restoreCounts = new int[sessionResources.size()];
for (int i = 0; i < sessionResources.size(); i++) {
restoreCounts[i] = sessionResources.get(i).getRestoreCount();
}
log.info("@@ Current counts {}", Arrays.toString(restoreCounts));
// Restart hive server
TestRemoteHiveDriver.stopHS2Service();
// Wait for server to stop
while (TestRemoteHiveDriver.getServerState() != Service.STATE.STOPPED) {
log.info("Waiting for HS2 to stop. Current state {}", TestRemoteHiveDriver.getServerState());
Thread.sleep(1000);
}
TestRemoteHiveDriver.createHS2Service();
// Wait for server to come up
while (Service.STATE.STARTED != TestRemoteHiveDriver.getServerState()) {
log.info("Waiting for HS2 to start {}", TestRemoteHiveDriver.getServerState());
Thread.sleep(1000);
}
Thread.sleep(10000);
log.info("Server restarted");
// Check params to be set
verifyParamOnRestart(lensSessionId);
// Poll for first query, we should not get any exception
ctx = waitForQueryToFinish(target(), lensSessionId, handle, defaultMT);
Assert.assertTrue(ctx.getStatus().finished());
log.info("Previous query status: {}", ctx.getStatus().getStatusMessage());
// After hive server restart, first few queries fail with Invalid Operation Handle followed by
// Invalid Session Handle. Ideal behaviour is to fail with Invalid Session Handle immediately.
// Jira Ticket raised for debugging: https://issues.apache.org/jira/browse/LENS-707
final String query = "select COUNT(ID) from test_hive_server_restart";
Response response = null;
while (true) {
response = execute(target(), Optional.of(lensSessionId), Optional.of(query), defaultMT);
if (response != null) {
LensAPIResult<QueryHandle> result = response.readEntity(new GenericType<LensAPIResult<QueryHandle>>() {});
handle = result.getData();
if (handle != null) {
break;
}
}
Thread.sleep(1000);
}
// Poll for second query, this should finish successfully
ctx = waitForQueryToFinish(target(), lensSessionId, handle, defaultMT);
log.info("Final status for {}: {}", handle, ctx.getStatus().getStatus());
// Now we can expect that session resources have been added back exactly once
for (int i = 0; i < sessionResources.size(); i++) {
LensSessionImpl.ResourceEntry resourceEntry = sessionResources.get(i);
//The restore count can vary based on How many Hive Drivers were able to execute the estimate on the query
//successfully after Hive Server Restart.
Assert.assertTrue((resourceEntry.getRestoreCount() > restoreCounts[i]
&& resourceEntry.getRestoreCount() <= restoreCounts[i] + NO_OF_HIVE_DRIVERS),
"Restore test failed for " + resourceEntry + " pre count=" + restoreCounts[i] + " post count=" + resourceEntry
.getRestoreCount());
log.info("@@ Latest count {}={}", resourceEntry, resourceEntry.getRestoreCount());
}
// Assert.assertEquals(stat.getStatus(), QueryStatus.Status.SUCCESSFUL,
// "Expected to be successful " + handle);
log.info("End hive server restart test");
LensServerTestUtil.dropTable("test_hive_server_restart", target(), lensSessionId, defaultMT);
queryService.closeSession(lensSessionId);
}
/**
* Test session restart.
*
* @throws Exception the exception
*/
@Test
public void testSessionRestart() throws Exception {
System.out.println("### Test session restart");
// Create a new session
WebTarget sessionTarget = target().path("session");
FormDataMultiPart sessionForm = new FormDataMultiPart();
sessionForm.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("username").build(), "foo"));
sessionForm.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("password").build(), "bar"));
sessionForm.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionconf").fileName("sessionconf")
.build(), new LensConf(), defaultMT));
final LensSessionHandle restartTestSession = sessionTarget.request(defaultMT).post(
Entity.entity(sessionForm, MediaType.MULTIPART_FORM_DATA_TYPE), LensSessionHandle.class);
Assert.assertNotNull(restartTestSession);
// Set a param
setParams(restartTestSession);
// Add resource
// add a resource
final WebTarget resourcetarget = target().path("session/resources");
final FormDataMultiPart mp1 = new FormDataMultiPart();
mp1.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), restartTestSession,
defaultMT));
mp1.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("type").build(), "file"));
mp1.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("path").build(),
"target/test-classes/lens-site.xml"));
APIResult result = resourcetarget.path("add").request(defaultMT)
.put(Entity.entity(mp1, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
assertEquals(result.getStatus(), Status.SUCCEEDED);
// restart server
restartLensServer(getServerConf());
// Check resources added again
verifyParamOnRestart(restartTestSession);
HiveSessionService sessionService = LensServices.get().getService(SessionService.NAME);
Assert.assertTrue(sessionService.getHealthStatus().isHealthy());
LensSessionImpl session = sessionService.getSession(restartTestSession);
assertEquals(session.getLensSessionPersistInfo().getResources().size(), 1);
LensSessionImpl.ResourceEntry resourceEntry = session.getLensSessionPersistInfo().getResources().get(0);
assertEquals(resourceEntry.getType(), "FILE");
Assert.assertTrue(resourceEntry.getUri().contains("target/test-classes/lens-site.xml"));
Assert.assertTrue(resourceEntry.getLocation().contains("target/test-classes/lens-site.xml"));
// close session
result = sessionTarget.queryParam("sessionid", restartTestSession).request(defaultMT).delete(APIResult.class);
assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
}
private void setParams(LensSessionHandle lensSessionHandle) {
FormDataMultiPart setpart = new FormDataMultiPart();
setpart.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionHandle,
defaultMT));
setpart
.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("key").build(), "lens.session.testRestartKey"));
setpart.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("value").build(), "myvalue"));
APIResult result = target().path("session").path("params").request(defaultMT)
.put(Entity.entity(setpart, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
}
private void verifyParamOnRestart(LensSessionHandle lensSessionHandle) {
StringList sessionParams = target().path("session").path("params").queryParam("sessionid", lensSessionHandle)
.queryParam("verbose", true).queryParam("key", "lens.session.testRestartKey").request(defaultMT)
.get(StringList.class);
System.out.println("Session params:" + sessionParams.getElements());
assertEquals(sessionParams.getElements().size(), 1);
Assert.assertTrue(sessionParams.getElements().contains("lens.session.testRestartKey=myvalue"));
}
@Test(dataProvider = "mediaTypeData")
public void testServerMustRestartOnManualDeletionOfAddedResources(MediaType mt)
throws IOException, LensException, LenServerTestException {
/* Begin: Setup */
/* Add a resource jar to current working directory */
File jarFile = new File(TestResourceFile.TEST_RESTART_ON_RESOURCE_MOVE_JAR.getValue());
FileUtils.touch(jarFile);
/* Add the created resource jar to lens server */
LensSessionHandle sessionHandle = LensServerTestUtil.openSession(target(), "foo", "bar", new LensConf(), mt);
LensServerTestUtil.addResource(target(), sessionHandle, "jar", jarFile.getPath(), mt);
/* Delete resource jar from current working directory */
LensServerTestFileUtils.deleteFile(jarFile);
/* End: Setup */
/* Verification Steps: server should restart without exceptions */
restartLensServer();
HiveSessionService service = LensServices.get().getService(SessionService.NAME);
service.closeSession(sessionHandle);
}
}