ElasticSearchRestClient.java

/**
 * Copyright (C) 2014 Stratio (http://stratio.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.stratio.ingestion.sink.elasticsearch.client;

import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import com.stratio.ingestion.sink.elasticsearch.ElasticSearchEventSerializer;
import com.stratio.ingestion.sink.elasticsearch.IndexNameBuilder;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.common.bytes.BytesReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

/**
 * Rest ElasticSearch client which is responsible for sending bulks of events to
 * ElasticSearch using ElasticSearch HTTP API. This is configurable, so any
 * config params required should be taken through this.
 */
public class  ElasticSearchRestClient implements ElasticSearchClient {

  private static final String INDEX_OPERATION_NAME = "index";
  private static final String INDEX_PARAM = "_index";
  private static final String TYPE_PARAM = "_type";
  private static final String TTL_PARAM = "_ttl";
  private static final String BULK_ENDPOINT = "_bulk";

  private static final Logger logger = LoggerFactory.getLogger(ElasticSearchRestClient.class);

  private final ElasticSearchEventSerializer serializer;
  private final RoundRobinList<String> serversList;
  
  private StringBuilder bulkBuilder;
  private HttpClient httpClient;
  
  public ElasticSearchRestClient(String[] hostNames,
      ElasticSearchEventSerializer serializer) {

    for (int i = 0; i < hostNames.length; ++i) {
      if (!hostNames[i].contains("http://") && !hostNames[i].contains("https://")) {
        hostNames[i] = "http://" + hostNames[i];
      }
    }
    this.serializer = serializer;

    serversList = new RoundRobinList<String>(Arrays.asList(hostNames));
    httpClient = new DefaultHttpClient();
    bulkBuilder = new StringBuilder();
  }

  @VisibleForTesting
  public ElasticSearchRestClient(String[] hostNames,
          ElasticSearchEventSerializer serializer, HttpClient client) {
    this(hostNames, serializer);
    httpClient = client;
  }

  @Override
  public void configure(Context context) {
  }

  @Override
  public void close() {
  }

  @Override
  public void addEvent(Event event, IndexNameBuilder indexNameBuilder, String indexType, long ttlMs) throws Exception {
    BytesReference content = serializer.getContentBuilder(event).bytes();
    Map<String, Map<String, String>> parameters = new HashMap<String, Map<String, String>>();
    Map<String, String> indexParameters = new HashMap<String, String>();
    indexParameters.put(INDEX_PARAM, indexNameBuilder.getIndexName(event));
    indexParameters.put(TYPE_PARAM, indexType);
    if (ttlMs > 0) {
      indexParameters.put(TTL_PARAM, Long.toString(ttlMs));
    }
    parameters.put(INDEX_OPERATION_NAME, indexParameters);

    Gson gson = new Gson();
    synchronized(bulkBuilder) {
      bulkBuilder.append(gson.toJson(parameters));
      bulkBuilder.append("\n");
      bulkBuilder.append(content.toBytesArray().toUtf8());
      bulkBuilder.append("\n");
    }
  }

  @Override
  public void execute() throws Exception {
    int statusCode = 0, triesCount = 0;
    HttpResponse response = null;
    String entity;
    synchronized (bulkBuilder) {
      entity = bulkBuilder.toString();
      bulkBuilder = new StringBuilder();
    }

    while (statusCode != HttpStatus.SC_OK && triesCount < serversList.size()) {
      triesCount++;
      String host = serversList.get();
      String url = host + "/" + BULK_ENDPOINT;
      HttpPost httpRequest = new HttpPost(url);
      httpRequest.setEntity(new StringEntity(entity));
      response = httpClient.execute(httpRequest);
      statusCode = response.getStatusLine().getStatusCode();
      logger.info("Status code from elasticsearch: " + statusCode);
      if (response.getEntity() != null)
        logger.debug("Status message from elasticsearch: " + EntityUtils.toString(response.getEntity(), "UTF-8"));
    }

    if (statusCode != HttpStatus.SC_OK) {
      if (response.getEntity() != null) {
        throw new EventDeliveryException(EntityUtils.toString(response.getEntity(), "UTF-8"));
      } else {
        throw new EventDeliveryException("Elasticsearch status code was: " + statusCode);
      }
    }
  }
}