/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lens.server;
import static org.apache.lens.server.LensServerTestUtil.DB_WITH_JARS;
import static org.apache.lens.server.LensServerTestUtil.DB_WITH_JARS_2;
import static org.apache.lens.server.LensServerTestUtil.createTestDatabaseResources;
import static org.testng.Assert.*;
import java.io.IOException;
import java.net.ServerSocket;
import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriBuilder;
import org.apache.lens.api.jaxb.LensJAXBContextResolver;
import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver;
import org.apache.lens.driver.hive.TestRemoteHiveDriver;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.metrics.LensMetricsUtil;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.query.QueryExecutionServiceImpl;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.Service;
import org.apache.hive.service.Service.STATE;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
import org.glassfish.jersey.moxy.json.MoxyJsonFeature;
import org.glassfish.jersey.test.JerseyTest;
import org.testng.annotations.AfterSuite;
import org.testng.annotations.BeforeSuite;
import org.testng.annotations.DataProvider;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
/**
* Extend this class for unit testing Lens Jersey resources
*/
@Slf4j
public abstract class LensJerseyTest extends JerseyTest {
private int port = -1;
protected MediaType defaultMT = MediaType.APPLICATION_XML_TYPE;
private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext();
protected URI getUri() {
return UriBuilder.fromUri("http://localhost/").port(getTestPort()).build();
}
private boolean isPortAlreadyFound() {
return port != -1;
}
public void setUp() throws Exception {
log.info("setUp in class: {}", this.getClass().getCanonicalName());
super.setUp();
}
public void tearDown() throws Exception {
log.info("tearDown in class: {}", this.getClass().getCanonicalName());
super.tearDown();
}
protected int getTestPort() {
if (isPortAlreadyFound()) {
return port;
}
ServerSocket socket = null;
try {
socket = new ServerSocket(0);
setPort(socket.getLocalPort());
} catch (IOException e) {
log.info("Exception occured while creating socket. Use a default port number {}", port);
} finally {
try {
if (socket != null) {
socket.close();
}
} catch (IOException e) {
log.info("Exception occured while closing the socket", e);
}
}
return port;
}
public void setPort(int localPort) {
port = localPort;
}
@Override
protected URI getBaseUri() {
return UriBuilder.fromUri(getUri()).path("lens-server").build();
}
@Override
protected void configureClient(ClientConfig config) {
config.register(MultiPartFeature.class);
config.register(MoxyJsonFeature.class);
config.register(MoxyJsonConfigurationContextResolver.class);
config.register(LensJAXBContextResolver.class);
}
/**
* This should used only on restarts and not during initial setup.
* @return Modified Server conf with over-writes
*/
public final HiveConf getServerConf() {
HiveConf serverConf = LensServerConf.getHiveConf();
Map<String, String> overWrites = getServerConfOverWrites();
if (overWrites != null) {
serverConf = new HiveConf(serverConf);
for (Map.Entry<String, String> overWrite : overWrites.entrySet()) {
serverConf.set(overWrite.getKey(), overWrite.getValue());
}
}
return serverConf;
}
public Map<String, String> getServerConfOverWrites() {
return null;
}
/**
* Start all.
*
* @throws Exception the exception
*/
@BeforeSuite
public void startAll() throws Exception {
log.info("Before suite");
System.setProperty("lens.log.dir", "target/");
System.setProperty(LensConfConstants.CONFIG_LOCATION, "target/test-classes/");
TestRemoteHiveDriver.createHS2Service();
System.out.println("Remote hive server started!");
HiveConf hiveConf = new HiveConf();
hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS, 5);
hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_CONNECTION_RETRY_LIMIT, 3);
hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_RETRY_LIMIT, 3);
createTestDatabaseResources(new String[]{DB_WITH_JARS, DB_WITH_JARS_2},
hiveConf);
LensServices.get().init(LensServerConf.getHiveConf());
LensServices.get().start();
// Check if mock service is started
Service mockSvc = LensServices.get().getService(MockNonLensService.NAME);
assertNotNull(mockSvc);
assertTrue(mockSvc instanceof MockNonLensService, mockSvc.getClass().getName());
assertEquals(mockSvc.getServiceState(), STATE.STARTED);
System.out.println("Lens services started!");
}
/**
* Stop all.
*
* @throws Exception the exception
*/
@AfterSuite
public void stopAll() throws Exception {
log.info("After suite");
verifyMetrics();
LensServices.get().stop();
System.out.println("Lens services stopped!");
TestRemoteHiveDriver.stopHS2Service();
System.out.println("Remote hive server stopped!");
}
/**
* Verify metrics.
*/
protected void verifyMetrics() {
// print final metrics
System.out.println("Final report");
MetricsService metrics = LensServices.get().getService(MetricsService.NAME);
metrics.publishReport();
// validate http error count
long httpClientErrors = metrics.getCounter(LensRequestListener.class, LensRequestListener.HTTP_CLIENT_ERROR);
long httpServerErrors = metrics.getCounter(LensRequestListener.class, LensRequestListener.HTTP_SERVER_ERROR);
long httpOtherErrors = metrics.getCounter(LensRequestListener.class, LensRequestListener.HTTP_UNKOWN_ERROR);
long httpErrors = metrics.getCounter(LensRequestListener.class, LensRequestListener.HTTP_ERROR);
assertEquals(httpClientErrors + httpServerErrors + httpOtherErrors, httpErrors,
"Server + Client error should equal total errors");
// validate http metrics
long httpReqStarted = metrics.getCounter(LensRequestListener.class, LensRequestListener.HTTP_REQUESTS_STARTED);
long httpReqFinished = metrics.getCounter(LensRequestListener.class, LensRequestListener.HTTP_REQUESTS_FINISHED);
assertEquals(httpReqStarted, httpReqFinished, "Total requests started should equal total requests finished");
// validate queries in the final state
long queriesSuccessful = metrics.getTotalSuccessfulQueries();
long queriesFailed = metrics.getTotalFailedQueries();
long queriesCancelled = metrics.getTotalCancelledQueries();
long queriesFinished = metrics.getTotalFinishedQueries();
assertEquals(queriesFinished, queriesSuccessful + queriesFailed + queriesCancelled,
"Total finished queries should be sum of successful, failed and cancelled queries");
}
/**
* Restart lens server.
*/
protected void restartLensServer() {
restartLensServer(LensServerConf.getHiveConf());
}
protected void restartLensServer(HiveConf conf) {
restartLensServer(conf, false);
}
/**
* Restart lens server.
*
* @param conf the conf
* @param pauseQuerySubmitter whether to pause query submitter while starting lens server
*/
public void restartLensServer(HiveConf conf, boolean pauseQuerySubmitter) {
LensServices.get().stop();
LensMetricsUtil.clearRegistry();
System.out.println("Lens services stopped!");
LensServices.setInstance(new LensServices(LensServices.LENS_SERVICES_NAME, this.logSegregationContext));
LensServices.get().init(conf);
if (pauseQuerySubmitter) {
QueryExecutionServiceImpl queryService = LensServices.get().getService(QueryExecutionService.NAME);
queryService.pauseQuerySubmitter(true);
System.out.println("Paused Query Submitter");
}
LensServices.get().start();
System.out.println("Lens services restarted!");
}
public static void waitForPurge(int allowUnpurgable,
ConcurrentLinkedQueue<QueryExecutionServiceImpl.FinishedQuery> finishedQueries) throws InterruptedException {
List<QueryExecutionServiceImpl.FinishedQuery> unPurgable = Lists.newArrayList();
for (QueryExecutionServiceImpl.FinishedQuery finishedQuery : finishedQueries) {
if (!finishedQuery.canBePurged()) {
unPurgable.add(finishedQuery);
}
}
if (unPurgable.size() > allowUnpurgable) {
throw new RuntimeException(unPurgable.size() + " finished queries can't be purged: " + unPurgable);
}
while (finishedQueries.size() > allowUnpurgable) {
Thread.sleep(5000);
}
}
@DataProvider(name = "mediaTypeData")
public Object[][] mediaTypeData() {
return new Object[][] {
{MediaType.APPLICATION_XML_TYPE},
{MediaType.APPLICATION_JSON_TYPE},
};
}
public static Entity getEntityForString(String o, MediaType mt) {
return Entity.entity(o, mt);
}
}