package fr.ippon.tatami.service.elasticsearch; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import fr.ippon.tatami.domain.Group; import fr.ippon.tatami.domain.User; import fr.ippon.tatami.domain.status.Status; import fr.ippon.tatami.repository.GroupDetailsRepository; import fr.ippon.tatami.service.SearchService; import org.apache.commons.lang.StringUtils; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.annotation.Cacheable; import org.springframework.scheduling.annotation.Async; import org.springframework.util.Assert; import javax.annotation.PostConstruct; import javax.inject.Inject; import java.io.IOException; import java.net.URL; import java.util.*; import static org.elasticsearch.index.query.FilterBuilders.termFilter; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; public class ElasticsearchSearchService implements SearchService { private static final Logger log = LoggerFactory.getLogger(ElasticsearchSearchService.class); private static final String ALL_FIELD = "_all"; private static final List<String> TYPES = Collections.unmodifiableList(Arrays.asList("user", "status", "group")); @Inject private ElasticsearchEngine engine; @Inject private String indexNamePrefix; @Inject private GroupDetailsRepository groupDetailsRepository; private Client client() { return engine.client(); } private String indexName(String type) { return StringUtils.isEmpty(indexNamePrefix) ? type : indexNamePrefix + '-' + type; } @PostConstruct private void init() { for (String type : TYPES) { if (!client().admin().indices().prepareExists(indexName(type)).execute().actionGet().exists()) { log.info("Index {} does not exists in Elasticsearch, creating it!", indexName(type)); createIndex(); } } } @Override public boolean reset() { log.info("Reseting ElasticSearch Index"); if (deleteIndex()) { return createIndex(); } else { log.warn("ElasticSearch Index could not be reset!"); return false; } } /** * Delete the tatami index. * * @return {@code true} if the index is deleted or didn't exist. */ private boolean deleteIndex() { for (String type : TYPES) { try { boolean ack = client().admin().indices().prepareDelete(indexName(type)).execute().actionGet().acknowledged(); if (!ack) { log.error("Elasticsearch Index wasn't deleted !"); return false; } } catch (IndexMissingException e) { // Failling to delete a missing index is supposed to be valid log.warn("Elasticsearch Index " + indexName(type) + " missing, it was not deleted"); } catch (ElasticSearchException e) { log.error("Elasticsearch Index " + indexName(type) + " was not deleted", e); return false; } } log.debug("Elasticsearch Index deleted!"); return true; } /** * Create the tatami index. * * @return {@code true} if an error occurs during the creation. */ private boolean createIndex() { for (String type : TYPES) { try { CreateIndexRequestBuilder createIndex = client().admin().indices().prepareCreate(indexName(type)); URL mappingUrl = getClass().getClassLoader().getResource("META-INF/elasticsearch/index/" + type + ".json"); ObjectMapper jsonMapper = new ObjectMapper(); JsonNode indexConfig = jsonMapper.readTree(mappingUrl); JsonNode indexSettings = indexConfig.get("settings"); if (indexSettings != null && indexSettings.isObject()) { createIndex.setSettings(jsonMapper.writeValueAsString(indexSettings)); } JsonNode mappings = indexConfig.get("mappings"); if (mappings != null && mappings.isObject()) { for (Iterator<Map.Entry<String, JsonNode>> i = mappings.fields(); i.hasNext(); ) { Map.Entry<String, JsonNode> field = i.next(); ObjectNode mapping = jsonMapper.createObjectNode(); mapping.put(field.getKey(), field.getValue()); createIndex.addMapping(field.getKey(), jsonMapper.writeValueAsString(mapping)); } } boolean ack = createIndex.execute().actionGet().acknowledged(); if (!ack) { log.error("Cannot create index " + indexName(type)); return false; } } catch (ElasticSearchException e) { log.error("Cannot create index " + indexName(type), e); return false; } catch (IOException e) { log.error("Cannot create index " + indexName(type), e); return false; } } return true; } private final ElasticsearchMapper<Status> statusMapper = new ElasticsearchMapper<Status>() { @Override public String id(Status status) { return status.getStatusId(); } @Override public String type() { return "status"; } @Override public String prefixSearchSortField() { return null; } @Override public XContentBuilder toJson(Status status) throws IOException { XContentBuilder source = XContentFactory.jsonBuilder() .startObject() .field("statusId", status.getStatusId()) .field("domain", status.getDomain()) .field("username", status.getUsername()) .field("statusDate", status.getStatusDate()) .field("content", status.getContent()); if (status.getGroupId() != null) { Group group = groupDetailsRepository.getGroupDetails(status.getGroupId()); source.field("groupId", status.getGroupId()); source.field("publicGroup", group.isPublicGroup()); } return source.endObject(); } }; @Override @Async public void addStatus(Status status) { index(status, statusMapper); } @Override public void addStatuses(Collection<Status> statuses) { indexAll(statuses, statusMapper); } @Override public void removeStatus(Status status) { Assert.notNull(status, "status cannot be null"); delete(status, statusMapper); } @Override public List<String> searchStatus(final String domain, final String query, int page, int size) { Assert.notNull(query); Assert.notNull(domain); if (page < 0) { page = 0; //Default value } if (size <= 0) { size = SearchService.DEFAULT_PAGE_SIZE; } try { SearchRequestBuilder searchRequest = client().prepareSearch(indexName(statusMapper.type())) .setTypes(statusMapper.type()) .setQuery(matchQuery(ALL_FIELD, query)) .setFilter(termFilter("domain", domain)) .addFields() .setFrom(page * size) .setSize(size) .addSort("statusDate", SortOrder.DESC); if (log.isTraceEnabled()) { log.trace("elasticsearch query : " + searchRequest); } SearchResponse searchResponse = searchRequest.execute().actionGet(); SearchHits searchHits = searchResponse.hits(); Long hitsNumber = searchHits.totalHits(); if (hitsNumber == 0) { return Collections.emptyList(); } SearchHit[] hits = searchHits.hits(); List<String> items = new ArrayList<String>(hits.length); for (SearchHit hit : hits) { items.add(hit.getId()); } log.debug("search status with words ({}) = {}", query, items); return items; } catch (IndexMissingException e) { log.warn("The index " + indexName(statusMapper.type()) + " was not found in the Elasticsearch cluster."); return Collections.emptyList(); } catch (ElasticSearchException e) { log.error("Error happened while searching status in index " + indexName(statusMapper.type())); return Collections.emptyList(); } } private final ElasticsearchMapper<User> userMapper = new ElasticsearchMapper<User>() { @Override public String id(User user) { return user.getLogin(); } @Override public String type() { return "user"; } @Override public String prefixSearchSortField() { return "username"; } @Override public XContentBuilder toJson(User user) throws IOException { return XContentFactory.jsonBuilder() .startObject() .field("login", user.getLogin()) .field("domain", user.getDomain()) .field("username", user.getUsername()) .field("firstName", user.getFirstName()) .field("lastName", user.getLastName()) .endObject(); } }; @Override @Async public void addUser(final User user) { Assert.notNull(user, "user cannot be null"); index(user, userMapper); } @Override public void addUsers(Collection<User> users) { indexAll(users, userMapper); } @Override public void removeUser(User user) { delete(user, userMapper); } @Override @Cacheable("user-prefix-cache") public Collection<String> searchUserByPrefix(String domain, String prefix) { return searchByPrefix(domain, prefix, DEFAULT_TOP_N_SEARCH_USER, userMapper); } private final ElasticsearchMapper<Group> groupMapper = new ElasticsearchMapper<Group>() { @Override public String id(Group group) { return group.getGroupId(); } @Override public String type() { return "group"; } @Override public String prefixSearchSortField() { return "name-not-analyzed"; } @Override public XContentBuilder toJson(Group group) throws IOException { return XContentFactory.jsonBuilder() .startObject() .field("domain", group.getDomain()) .field("groupId", group.getGroupId()) .field("name", group.getName()) .field("description", group.getDescription()) .endObject(); } }; @Override @Async public void addGroup(Group group) { index(group, groupMapper); } @Override public void removeGroup(Group group) { delete(group, groupMapper); } @Override @Cacheable("group-prefix-cache") public Collection<Group> searchGroupByPrefix(String domain, String prefix, int size) { Collection<String> ids = searchByPrefix(domain, prefix, size, groupMapper); List<Group> groups = new ArrayList<Group>(ids.size()); for (String id : ids) { groups.add(groupDetailsRepository.getGroupDetails(id)); } return groups; } /** * Indexes an object to elasticsearch. * This method is asynchronous. * * @param object Object to index. * @param mapper Converter to JSON. */ private <T> void index(T object, ElasticsearchMapper<T> mapper) { Assert.notNull(object); Assert.notNull(mapper); final String type = mapper.type(); final String id = mapper.id(object); try { final XContentBuilder source = mapper.toJson(object); log.debug("Ready to index the {} id {} into Elasticsearch: {}", type, id, stringify(source)); client().prepareIndex(indexName(type), type, id).setSource(source).execute(new ActionListener<IndexResponse>() { @Override public void onResponse(IndexResponse response) { log.debug(type + " id " + id + " was " + (response.version() == 1 ? "indexed" : "updated") + " into Elasticsearch"); } @Override public void onFailure(Throwable e) { log.error("The " + type + " id " + id + " wasn't indexed : " + stringify(source), e); } }); } catch (IOException e) { log.error("The " + type + " id " + id + " wasn't indexed", e); } } /** * Indexes an collection of objects to elasticsearch. * This method is synchronous. * * @param collection Object to index. * @param adapter Converter to JSON. */ private <T> void indexAll(Collection<T> collection, ElasticsearchMapper<T> adapter) { Assert.notNull(collection); Assert.notNull(adapter); if (collection.isEmpty()) return; String type = adapter.type(); BulkRequestBuilder request = client().prepareBulk(); for (T object : collection) { String id = adapter.id(object); try { XContentBuilder source = adapter.toJson(object); IndexRequestBuilder indexRequest = client().prepareIndex(indexName(type), type, id).setSource(source); request.add(indexRequest); } catch (IOException e) { log.error("The " + type + " of id " + id + " wasn't indexed", e); } } log.debug("Ready to index {} {} into Elasticsearch.", collection.size(), type); BulkResponse response = request.execute().actionGet(); if (response.hasFailures()) { int errorCount = 0; for (BulkItemResponse itemResponse : response) { if (itemResponse.failed()) { log.error("The " + type + " of id " + itemResponse.getId() + " wasn't indexed in bulk operation: " + itemResponse.getFailureMessage()); ++errorCount; } } log.error(errorCount + " " + type + " where not indexed in bulk operation."); } else { log.debug("{} {} indexed into Elasticsearch in bulk operation.", collection.size(), type); } } /** * delete a document. * This method is asynchronous. * * @param object Object to index. * @param mapper Converter to JSON. */ private <T> void delete(T object, ElasticsearchMapper<T> mapper) { Assert.notNull(object); Assert.notNull(mapper); final String id = mapper.id(object); final String type = mapper.type(); log.debug("Ready to delete the {} of id {} from Elasticsearch: ", type, id); client().prepareDelete(indexName(type), type, id).execute(new ActionListener<DeleteResponse>() { @Override public void onResponse(DeleteResponse deleteResponse) { if (log.isDebugEnabled()) { if (deleteResponse.notFound()) { log.debug("{} of id {} was not found therefore not deleted.", type, id); } else { log.debug("{} of id {} was deleted from Elasticsearch.", type, id); } } } @Override public void onFailure(Throwable e) { log.error("The " + type + " of id " + id + " wasn't deleted from Elasticsearch.", e); } }); } private Collection<String> searchByPrefix(String domain, String prefix, int size, ElasticsearchMapper<?> mapper) { try { SearchRequestBuilder searchRequest = client().prepareSearch(indexName(mapper.type())) .setTypes(mapper.type()) .setQuery(matchQuery("prefix", prefix)) .setFilter(termFilter("domain", domain)) .addFields() .setFrom(0) .setSize(size) .addSort(SortBuilders.fieldSort(mapper.prefixSearchSortField()).order(SortOrder.ASC)); if (log.isTraceEnabled()) { log.trace("elasticsearch query : " + searchRequest); } SearchResponse searchResponse = searchRequest .execute() .actionGet(); SearchHits searchHits = searchResponse.hits(); if (searchHits.totalHits() == 0) return Collections.emptyList(); SearchHit[] hits = searchHits.hits(); final List<String> ids = new ArrayList<String>(hits.length); for (SearchHit hit : hits) { ids.add(hit.getId()); } log.debug("search " + mapper.type() + " by prefix(\"" + domain + "\", \"" + prefix + "\") = result : " + ids); return ids; } catch (IndexMissingException e) { log.warn("The index " + indexName(mapper.type()) + " was not found in the Elasticsearch cluster."); return Collections.emptyList(); } catch (ElasticSearchException e) { log.error("Error while searching user by prefix in index " + indexName(mapper.type()), e); return Collections.emptyList(); } } /** * Stringify a document source for logging purpose. * * @param source Source of the document. * @return A string representation of the document only valid for logging purpose. */ private String stringify(XContentBuilder source) { try { return source.prettyPrint().string(); } catch (IOException e) { return ""; } } /** * Used to transform an object to it's indexed representation. */ private static interface ElasticsearchMapper<T> { /** * Provides object id; * * @param o object. * @return object id. */ String id(T o); /** * Provides index type of this mapping. * * @return The elasticsearch index type of the object. */ String type(); /** * @return The name of the field to sort by in search by prefix. */ String prefixSearchSortField(); /** * Convert object to it's indexable JSON document representation. * * @param o object. * @return Document * @throws IOException If the creation of the JSON document failed. */ XContentBuilder toJson(T o) throws IOException; } }