LogAnalysisAppTest.java

/*
 * Copyright © 2015 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.cdap.examples.loganalysis;

import co.cask.cdap.test.ApplicationManager;
import co.cask.cdap.test.MapReduceManager;
import co.cask.cdap.test.ServiceManager;
import co.cask.cdap.test.SparkManager;
import co.cask.cdap.test.StreamManager;
import co.cask.cdap.test.TestBase;
import co.cask.cdap.test.TestConfiguration;
import co.cask.common.http.HttpRequest;
import co.cask.common.http.HttpRequests;
import co.cask.common.http.HttpResponse;
import com.google.common.collect.ImmutableMap;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Map;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;

/**
 * Unit test for {@link LogAnalysisApp}
 */
public class LogAnalysisAppTest extends TestBase {

  private static final Gson GSON = new Gson();
  private static final String LOG_1 = "127.0.0.1 - - [21/Jul/2014:9:55:27 -0800] \"GET /home.html HTTP/1.1\" 200 2048";
  private static final String LOG_2 = "127.0.1.1 - - [21/Jul/2014:9:55:27 -0800] \"GET /home.html HTTP/1.1\" 400 2048";
  private static final String LOG_3 = "127.0.0.1 - - [21/Jul/2014:9:55:27 -0800] \"GET /index.html HTTP/1.1\" 200 2048";

  private static final String TOTAL_HITS_VALUE = "2";
  private static final String TOTAL_RESPONSE_VALUE = "2";
  private static final String RESPONSE_CODE = "200";
  private static final Map<String, Integer> TPFS_RESULT = ImmutableMap.of("127.0.1.1", 1, "127.0.0.1", 2);

  @ClassRule
  public static final TestConfiguration CONFIG = new TestConfiguration("explore.enabled", false);

  @Test
  public void test() throws Exception {
    // Deploy the App
    ApplicationManager appManager = deployApplication(LogAnalysisApp.class);

    // Send a stream events to the Stream
    StreamManager streamManager = getStreamManager(LogAnalysisApp.LOG_STREAM);
    streamManager.send(LOG_1);
    streamManager.send(LOG_2);
    streamManager.send(LOG_3);

    // run the spark program
    SparkManager sparkManager = appManager.getSparkManager(
      LogAnalysisApp.ResponseCounterSpark.class.getSimpleName()).start();
    sparkManager.waitForFinish(60, TimeUnit.SECONDS);

    // run the mapreduce job
    MapReduceManager mapReduceManager = appManager.getMapReduceManager(HitCounterProgram.class.getSimpleName()).start();
    mapReduceManager.waitForFinish(3, TimeUnit.MINUTES);

    // start and wait for services
    ServiceManager hitCounterServiceManager = getServiceManager(appManager, LogAnalysisApp.HIT_COUNTER_SERVICE);
    ServiceManager responseCounterServiceManager = getServiceManager(appManager,
                                                                     LogAnalysisApp.RESPONSE_COUNTER_SERVICE);
    ServiceManager requestCounterServiceManager = getServiceManager(appManager,
                                                                    LogAnalysisApp.REQUEST_COUNTER_SERVICE);

    //Query for hit counts and verify it
    URL totalHitsURL = new URL(hitCounterServiceManager.getServiceURL(15, TimeUnit.SECONDS),
                               LogAnalysisApp.HitCounterServiceHandler.HIT_COUNTER_SERVICE_PATH);

    HttpResponse response = HttpRequests.execute(HttpRequest.post(totalHitsURL)
                                                   .withBody("{\"url\":\"" + "/home.html" + "\"}")
                                                   .build());
    Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
    Assert.assertEquals(TOTAL_HITS_VALUE, response.getResponseBodyAsString());

    // query for total responses for a response code and verify it
    URL responseCodeURL = new URL(responseCounterServiceManager.getServiceURL(15, TimeUnit.SECONDS),
                                  LogAnalysisApp.ResponseCounterHandler.RESPONSE_COUNT_PATH
                                    + "/" + RESPONSE_CODE);
    HttpRequest request = HttpRequest.get(responseCodeURL).build();
    response = HttpRequests.execute(request);
    Assert.assertEquals(TOTAL_RESPONSE_VALUE, response.getResponseBodyAsString());

    // query to get partitions in the request count tpfs
    URL requestCountFilsetsURL = new URL(requestCounterServiceManager.getServiceURL(15, TimeUnit.SECONDS),
                                         LogAnalysisApp.RequestCounterHandler.REQUEST_COUNTER_PARTITIONS_PATH);
    request = HttpRequest.get(requestCountFilsetsURL).build();
    response = HttpRequests.execute(request);
    TreeSet<String> partitions = GSON.fromJson(response.getResponseBodyAsString(),
                                           new TypeToken<TreeSet<String>>() {
                                           }.getType());
    Assert.assertEquals(1, partitions.size());
    String partition = partitions.iterator().next();

    //Query for the contents of the files in this partition and verify
    URL requestFilesetContentURL = new URL(requestCounterServiceManager.getServiceURL(15, TimeUnit.SECONDS),
                                           LogAnalysisApp.RequestCounterHandler.REQUEST_FILE_CONTENT_PATH);

    response = HttpRequests.execute(HttpRequest.post(requestFilesetContentURL)
                                      .withBody("{\"" +
                                                  LogAnalysisApp.RequestCounterHandler.REQUEST_FILE_PATH_HANDLER_KEY +
                                                  "\":\"" + partition + "\"}")
                                      .build());
    Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
    Map<String, Integer> responseMap = GSON.fromJson(response.getResponseBodyAsString(),
                                                     new TypeToken<Map<String, Integer>>() { }.getType());
    Assert.assertTrue(responseMap.equals(TPFS_RESULT));
  }

  private ServiceManager getServiceManager(ApplicationManager appManager, String serviceName)
    throws InterruptedException {
    // Start the service
    ServiceManager serviceManager =
      appManager.getServiceManager(serviceName).start();

    // Wait for service startup
    serviceManager.waitForStatus(true);
    return serviceManager;
  }
}