package com.cadrlife.devsearch.esplugin.service;
import com.cadrlife.devsearch.domain.Project;
import com.cadrlife.devsearch.esplugin.domain.*;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.base.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.name.Named;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.index.query.AndFilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.index.query.QueryStringQueryBuilder.Operator;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.facet.FacetBuilders;
import org.elasticsearch.search.facet.terms.TermsFacet;
import org.elasticsearch.search.sort.SortOrder;
import java.util.*;
public class CodeSearchIndexService {
private static final int MAX_RESULTS = 10000;
private static final ESLogger LOG = ESLoggerFactory.getLogger(CodeSearchIndexService.class.getName());
public static final String DOC_TYPE = "doc";
public static final String PROJECT_TYPE = "project";
private final String codeIndex;
private Client esClient;
@Inject
public CodeSearchIndexService(Client esClient, @Named("codeIndex") String codeIndex) {
this.esClient = esClient;
this.codeIndex = codeIndex;
}
public SearchResponse search(String query, int from, int max, String repo, String extension) {
QueryStringQueryBuilder queryBuilder = QueryBuilders.queryString(query).defaultField("content").defaultOperator(Operator.AND);
SearchRequestBuilder requestBuilder = esClient.prepareSearch(codeIndex)
.setTypes(DOC_TYPE)
.setFrom(from).setSize(max)
// .setHighlighterPreTags("<span class='hilight'>").setHighlighterPostTags("</span>")
// .setHighlighterEncoder("html")
// .addHighlightedField("content")
// .addSort("project",SortOrder.ASC)
// .addSort("repo",SortOrder.ASC)
.setQuery(queryBuilder)
.addFacet(FacetBuilders.termsFacet("extension").field("extension"))
.addFacet(FacetBuilders.termsFacet("repo").field("repo"));
AndFilterBuilder filter = FilterBuilders.andFilter();
requestBuilder.setPostFilter(filter);
if (!Strings.isNullOrEmpty(extension) && !"All".equals(extension)) {
filter.add(FilterBuilders.termFilter("extension", extension));
}
if (!Strings.isNullOrEmpty(repo) && !"All".equals(repo)) {
filter.add(FilterBuilders.termFilter("repo", repo));
}
ActionFuture<SearchResponse> search = requestBuilder.execute();
return checkResponseForErrors(query, search.actionGet());
}
public SearchResponse searchWithinField(String query, String field, int from, int max) {
ActionFuture<SearchResponse> search = esClient.prepareSearch(codeIndex)
.setTypes(DOC_TYPE)
.setFrom(from).setSize(max)
.setQuery(QueryBuilders.queryString(query).defaultField(field))
// Sorting used for suggest feature.
.addSort("project", SortOrder.ASC)
.addSort("repo", SortOrder.ASC)
.execute();
return checkResponseForErrors(query, search.actionGet());
}
private SearchResponse checkResponseForErrors(String query, SearchResponse response) {
for (ShardSearchFailure failure : response.getShardFailures()) {
LOG.error("Shard failure with query {}: {}", query, failure.reason());
}
// Possibly throw an exception here.
return response;
}
public ProjectList findAllProjects(int from, int max) {
SearchRequestBuilder searchRequestBuilder = esClient.prepareSearch(codeIndex).setTypes(PROJECT_TYPE);
// actionGet.addSort("name", SortOrder.ASC);
SearchResponse searchResponse = searchRequestBuilder
.setQuery(QueryBuilders.matchAllQuery())
.addSort("name", SortOrder.ASC)
.addSort("repo", SortOrder.ASC)
.setFrom(from).setSize(max).execute().actionGet();
//LOG.warn(searchResponse.toString());
// actionGet.ge
// SearchSourceBuilder searchSource = SearchSourceBuilder.searchSource();
// searchSource.from(skip).size(limit);
// searchSource.query(new TermQueryBuilder("content","LinkedHashMap"));
// searchSource.query(QueryBuilders.termQuery("type", "project"));
ProjectList projectList = new ProjectList();
for (SearchHit hit : searchResponse.getHits()) {
String name = getField(hit, "name");
String repo = getField(hit, "repo");
String lastIndexed = getField(hit, "lastIndexed");
String lastChanged = getField(hit, "lastChanged");
Project project = new Project().setName(name).setRepo(repo).setProjectType("unknown").setLastIndexed(lastIndexed).setLastChanged(lastChanged);
projectList.getProjects().add(project);
}
return projectList;
}
public SearchResultItem loadDoc(String path) {
GetResponse getResponse = esClient.prepareGet(codeIndex, DOC_TYPE, path).execute().actionGet();
String content = getField(getResponse, "content");
String project = getField(getResponse, "project");
String repo = getField(getResponse, "repo");
String filePath = getField(getResponse, "filePath");
String baseFilename = getField(getResponse, "baseFilename");
return new SearchResultItem()
.setPreview(content)
.setRepo(repo)
.setProject(project)
.setFilePath(filePath)
.setBaseFilename(baseFilename)
.setSourceReferences(collectSourceReferences(getResponse.getSource()));
}
private List<SourceReference> collectSourceReferences(Map<String, Object> source) {
Object references = source.get("references");
List<SourceReference> result = new ArrayList<>();
if (references instanceof Iterable) {
for (String refString : (Iterable<String>)references) {
try {
int i = refString.indexOf("::");
String id = i < 0 ? "" : refString.substring(i + 2);
String desc = i < 0 ? refString : refString.substring(0, i);
int lastDotIndex = desc.lastIndexOf(".");
String className = desc.substring(lastDotIndex + 1);
result.add(new SourceReference().setId(id).setDescription(desc).setMatchKey(className));
}
catch (StringIndexOutOfBoundsException e) {
throw new RuntimeException("Failed to parse ref string '" + refString + "'", e);
}
}
}
return result;
}
public Project loadProject(String path) {
GetResponse getResponse = esClient.prepareGet(codeIndex, PROJECT_TYPE, path).execute().actionGet();
String name = getField(getResponse, "name");
String repo = getField(getResponse, "repo");
String lastIndexed = getField(getResponse, "lastIndexed");
String lastChanged = getField(getResponse, "lastChanged");
return new Project().setRepo(repo).setName(name).setLastIndexed(lastIndexed).setLastChanged(lastChanged);
}
private String getField(GetResponse getResponse, String field) {
Object val = getResponse.getSource().get(field);
return val == null ? null : val.toString();
}
private String getField(SearchHit hit, String field) {
Object val = hit.getSource().get(field);
return val == null ? null : val.toString();
}
public List <String> findAllDocPathsForProject(String repo, String projectName) {
LOG.info("findAllDocPathsForProject '{}', '{}'", repo, projectName);
SearchResponse response = esClient
.prepareSearch(codeIndex)
.setTypes(DOC_TYPE)
.setQuery(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("project", projectName)).must(QueryBuilders.termQuery("repo", repo)))
.addFields("filePath")
.setFrom(0).setSize(MAX_RESULTS)
.execute()
.actionGet();
// .prepareGet(CODE_INDEX, DOC_TYPE, projectId(repo, projectName)+"*").execute().actionGet();
LOG.info("findAllDocPathsForProject '{}', '{}' totalHits {} fails {}", repo, projectName, response.getHits().getTotalHits(), response.getFailedShards());
return convertDocSearchResultToFilenameList(response);
}
// List<String> convertDocSearchResultToFilenameList(GetResponse response) {
// List <String> names = new ArrayList<>();
// for (SearchHit hit : response.get()) {
// names.add(hit.getSource().get("filePath").toString());
// }
// return names;
// }
private List<String> convertDocSearchResultToFilenameList(SearchResponse response) {
List <String> names = new ArrayList<>();
for (SearchHit hit : response.getHits()) {
names.add(hit.field("filePath").getValue().toString());
}
Collections.sort(names);
return names;
}
public DevSearchStatus getStatus() {
DevSearchStatus status = new DevSearchStatus();
status.setStatus("ok");
CountResponse countResponse = esClient.prepareCount(codeIndex)
.setTypes(DOC_TYPE)
.setQuery(QueryBuilders.matchAllQuery())
.execute()
.actionGet();
status.setDocCount(countResponse.getCount());
TermsFacet docFacet = doRepoFacetSearch(DOC_TYPE).getFacets().facet("repo");
Map<String, RepoStatus> repoStatusMap = new TreeMap<>();
for (TermsFacet.Entry e : docFacet.getEntries()) {
String name = e.getTerm().string();
repoStatusMap.put(name.toLowerCase(), new RepoStatus().setName(name).setDocCount(e.getCount()));
}
TermsFacet projectFacet = doRepoFacetSearch(PROJECT_TYPE).getFacets().facet("repo");
for (TermsFacet.Entry e : projectFacet.getEntries()) {
String name = e.getTerm().string();
RepoStatus repoStatus = repoStatusMap.get(name.toLowerCase());
if (repoStatus == null) {
repoStatus = new RepoStatus().setName(name);
}
repoStatusMap.put(name.toLowerCase(), repoStatus.setProjectCount(e.getCount()));
}
status.getRepos().addAll(repoStatusMap.values());
return status;
}
private SearchResponse doRepoFacetSearch(String type) {
int numberHigherThanRepoCount = 999;
return esClient.prepareSearch(codeIndex)
.setTypes(type).setSize(0)
.setQuery(QueryBuilders.matchAllQuery())
.addFacet(FacetBuilders.termsFacet("repo").field("repo").size(numberHigherThanRepoCount))
.execute()
.actionGet();
}
}