package fi.otavanopisto.muikku.plugins.search;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
import static org.elasticsearch.index.query.QueryBuilders.idsQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.IdsQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.sort.SortOrder;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoField;
import fi.otavanopisto.muikku.controller.PluginSettingsController;
import fi.otavanopisto.muikku.model.users.EnvironmentRoleArchetype;
import fi.otavanopisto.muikku.model.workspace.WorkspaceAccess;
import fi.otavanopisto.muikku.model.workspace.WorkspaceEntity;
import fi.otavanopisto.muikku.schooldata.SchoolDataIdentifier;
import fi.otavanopisto.muikku.schooldata.WorkspaceEntityController;
import fi.otavanopisto.muikku.search.SearchProvider;
import fi.otavanopisto.muikku.search.SearchResult;
@ApplicationScoped
public class ElasticSearchProvider implements SearchProvider {
@Inject
private Logger logger;
@Inject
private WorkspaceEntityController workspaceEntityController;
@Inject
private PluginSettingsController pluginSettingsController;
@Override
public void init() {
String clusterName = pluginSettingsController.getPluginSetting("elastic-search", "clusterName");
if (clusterName == null) {
clusterName = System.getProperty("elasticsearch.cluster.name");
}
if (clusterName == null) {
clusterName = "elasticsearch";
}
String portNumberProperty = System.getProperty("elasticsearch.node.port");
int portNumber;
if (portNumberProperty != null) {
portNumber = Integer.decode(portNumberProperty);
} else {
portNumber = 9300;
}
Settings settings = Settings.settingsBuilder()
.put("cluster.name", clusterName).build();
try {
elasticClient = TransportClient.builder().settings(settings).build()
.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), portNumber));
} catch (UnknownHostException e) {
logger.log(Level.SEVERE, "Failed to connect to elasticsearch cluster", e);
return;
}
}
@Override
public void deinit() {
elasticClient.close();
//node.close();
}
private String sanitizeSearchString(String query) {
if (query == null)
return null;
// TODO: query_string search for case insensitive searches??
// http://stackoverflow.com/questions/17266830/case-insensitivity-does-not-work
String ret = query.toLowerCase();
// Replace characters we don't support at the moment
ret = ret.replace('-', ' ');
ret = ret.trim();
return ret;
}
@Override
public SearchResult searchUsers(String text, String[] textFields, Collection<EnvironmentRoleArchetype> archetypes,
Collection<Long> groups, Collection<Long> workspaces, Collection<SchoolDataIdentifier> userIdentifiers,
Boolean includeInactiveStudents, Boolean includeHidden, Boolean onlyDefaultUsers, int start, int maxResults,
Collection<String> fields, Collection<SchoolDataIdentifier> excludeSchoolDataIdentifiers,
Date startedStudiesBefore, Date studyTimeEndsBefore) {
try {
long now = OffsetDateTime.now().toEpochSecond();
text = sanitizeSearchString(text);
BoolQueryBuilder query = boolQuery();
if (!Boolean.TRUE.equals(includeHidden)) {
query.mustNot(termQuery("hidden", true));
}
if (Boolean.TRUE.equals(onlyDefaultUsers)) {
query.must(termQuery("isDefaultIdentifier", true));
}
if (StringUtils.isNotBlank(text) && !ArrayUtils.isEmpty(textFields)) {
String[] words = text.split(" ");
for (int i = 0; i < words.length; i++) {
if (StringUtils.isNotBlank(words[i])) {
BoolQueryBuilder fieldBuilder = boolQuery();
for (String textField : textFields) {
fieldBuilder.should(prefixQuery(textField, words[i]));
}
query.must(fieldBuilder);
}
}
}
if (excludeSchoolDataIdentifiers != null) {
IdsQueryBuilder excludeIdsQuery = idsQuery("User");
for (SchoolDataIdentifier excludeSchoolDataIdentifier : excludeSchoolDataIdentifiers) {
excludeIdsQuery.addIds(String.format("%s/%s", excludeSchoolDataIdentifier.getIdentifier(), excludeSchoolDataIdentifier.getDataSource()));
}
query.mustNot(excludeIdsQuery);
}
if (startedStudiesBefore != null ) {
query.must(rangeQuery("studyStartDate").lt((long) startedStudiesBefore.getTime() / 1000));
}
if (studyTimeEndsBefore != null) {
query.must(rangeQuery("studyTimeEnd").lt((long) studyTimeEndsBefore.getTime() / 1000));
}
if (archetypes != null) {
List<String> archetypeNames = new ArrayList<>(archetypes.size());
for (EnvironmentRoleArchetype archetype : archetypes) {
archetypeNames.add(archetype.name().toLowerCase());
}
query.must(termsQuery("archetype", archetypeNames.toArray(new String[0])));
}
if (groups != null) {
query.must(termsQuery("groups", ArrayUtils.toPrimitive(groups.toArray(new Long[0]))));
}
if (workspaces != null) {
query.must(termsQuery("workspaces", ArrayUtils.toPrimitive(workspaces.toArray(new Long[0]))));
}
if (userIdentifiers != null) {
IdsQueryBuilder includeIdsQuery = idsQuery("User");
for (SchoolDataIdentifier userIdentifier : userIdentifiers) {
includeIdsQuery.addIds(String.format("%s/%s", userIdentifier.getIdentifier(), userIdentifier.getDataSource()));
}
query.must(includeIdsQuery);
}
if (includeInactiveStudents == false) {
/**
* List only active users.
*
* Active user is
* - staff member (TEACHER, MANAGER, ADMINISTRATOR, STUDY PROGRAMME LEADER) or
* - student that has study start date (in the past) and no study end date
* - student that has study start date (in the past) and study end date in the future
* - student that has no study start and end date but belongs to an active workspace
*
* Active workspace is
* - published and
* - either has no start/end date or current date falls between them
*/
Set<Long> activeWorkspaceEntityIds = getActiveWorkspaces();
query.must(
boolQuery()
.should(termsQuery("archetype",
EnvironmentRoleArchetype.TEACHER.name().toLowerCase(),
EnvironmentRoleArchetype.MANAGER.name().toLowerCase(),
EnvironmentRoleArchetype.STUDY_PROGRAMME_LEADER.name().toLowerCase(),
EnvironmentRoleArchetype.ADMINISTRATOR.name().toLowerCase())
)
.should(boolQuery()
.must(termQuery("archetype", EnvironmentRoleArchetype.STUDENT.name().toLowerCase()))
.must(existsQuery("studyStartDate"))
.must(rangeQuery("studyStartDate").lte(now))
.mustNot(existsQuery("studyEndDate"))
)
.should(boolQuery()
.must(termQuery("archetype", EnvironmentRoleArchetype.STUDENT.name().toLowerCase()))
.must(existsQuery("studyStartDate"))
.must(rangeQuery("studyStartDate").lte(now))
.must(existsQuery("studyEndDate"))
.must(rangeQuery("studyEndDate").gte(now))
)
.should(boolQuery()
.must(termQuery("archetype", EnvironmentRoleArchetype.STUDENT.name().toLowerCase()))
.mustNot(existsQuery("studyEndDate"))
.mustNot(existsQuery("studyStartDate"))
.must(termsQuery("workspaces", ArrayUtils.toPrimitive(activeWorkspaceEntityIds.toArray(new Long[0]))))
)
);
}
SearchRequestBuilder requestBuilder = elasticClient
.prepareSearch("muikku")
.setTypes("User")
.setFrom(start)
.setSize(maxResults);
if (CollectionUtils.isNotEmpty(fields)) {
requestBuilder.addFields(fields.toArray(new String[0]));
}
SearchResponse response = requestBuilder
.setQuery(query)
.addSort("_score", SortOrder.DESC)
.addSort("lastName", SortOrder.ASC)
.addSort("firstName", SortOrder.ASC)
.execute()
.actionGet();
List<Map<String, Object>> searchResults = new ArrayList<Map<String, Object>>();
SearchHit[] results = response.getHits().getHits();
for (SearchHit hit : results) {
Map<String, Object> hitSource = hit.getSource();
if(hitSource == null){
hitSource = new HashMap<>();
for(String key : hit.getFields().keySet()){
hitSource.put(key, hit.getFields().get(key).getValue().toString());
}
}
hitSource.put("indexType", hit.getType());
searchResults.add(hitSource);
}
SearchResult result = new SearchResult(searchResults.size(), start, maxResults, searchResults);
return result;
} catch (Exception e) {
logger.log(Level.SEVERE, "ElasticSearch query failed unexpectedly", e);
return new SearchResult(0, 0, 0, new ArrayList<Map<String,Object>>());
}
}
@Override
public SearchResult searchUsers(String text, String[] textFields, Collection<EnvironmentRoleArchetype> archetypes,
Collection<Long> groups, Collection<Long> workspaces, Collection<SchoolDataIdentifier> userIdentifiers,
Boolean includeInactiveStudents, Boolean includeHidden, Boolean onlyDefaultUsers, int start, int maxResults,
Collection<String> fields, Collection<SchoolDataIdentifier> excludeSchoolDataIdentifiers, Date startedStudiesBefore) {
return searchUsers(text, textFields, archetypes, groups, workspaces, userIdentifiers, includeInactiveStudents, includeHidden,
onlyDefaultUsers, start, maxResults, fields, excludeSchoolDataIdentifiers, startedStudiesBefore, null);
}
@Override
public SearchResult searchUsers(String text, String[] textFields, Collection<EnvironmentRoleArchetype> archetypes,
Collection<Long> groups, Collection<Long> workspaces, Collection<SchoolDataIdentifier> userIdentifiers,
Boolean includeInactiveStudents, Boolean includeHidden, Boolean onlyDefaultUsers, int start, int maxResults) {
return searchUsers(text, textFields, archetypes, groups, workspaces, userIdentifiers, includeInactiveStudents, includeHidden,
onlyDefaultUsers, start, maxResults, null);
}
@Override
public SearchResult searchUsers(String text, String[] textFields, Collection<EnvironmentRoleArchetype> archetypes,
Collection<Long> groups, Collection<Long> workspaces, Collection<SchoolDataIdentifier> userIdentifiers,
Boolean includeInactiveStudents, Boolean includeHidden, Boolean onlyDefaultUsers, int start, int maxResults, Collection<String> fields) {
return searchUsers(text, textFields, archetypes, groups, workspaces, userIdentifiers, includeInactiveStudents, includeHidden,
onlyDefaultUsers, start, maxResults, fields, null, null);
}
private Set<Long> getActiveWorkspaces() {
long now = OffsetDateTime.now().with(ChronoField.MILLI_OF_DAY, 0).toInstant().toEpochMilli() / 1000;
BoolQueryBuilder query = boolQuery();
query.must(termQuery("published", Boolean.TRUE));
query.must(
boolQuery()
.should(boolQuery()
.mustNot(existsQuery("beginDate"))
.mustNot(existsQuery("endDate"))
)
.should(boolQuery()
.must(existsQuery("beginDate"))
.must(existsQuery("endDate"))
.must(rangeQuery("beginDate").lte(now))
.must(rangeQuery("endDate").gte(now))
)
);
SearchResponse response = elasticClient
.prepareSearch("muikku")
.setTypes("Workspace")
.setQuery(query)
.setNoFields()
.setSize(Integer.MAX_VALUE)
.execute()
.actionGet();
SearchHit[] hits = response.getHits().getHits();
Set<SchoolDataIdentifier> identifiers = new HashSet<>();
for (SearchHit hit : hits) {
String[] id = hit.getId().split("/", 2);
if (id.length == 2) {
String dataSource = id[1];
String identifier = id[0];
identifiers.add(new SchoolDataIdentifier(identifier, dataSource));
}
}
return workspaceEntityController.findWorkspaceEntityIdsByIdentifiers(identifiers);
}
@Override
public SearchResult searchWorkspaces(String schoolDataSource, List<String> subjects, List<String> identifiers, String freeText, boolean includeUnpublished, int start, int maxResults) {
return searchWorkspaces(schoolDataSource, subjects, identifiers, null, null, freeText, null, null, includeUnpublished, start, maxResults, null);
}
@Override
public SearchResult searchWorkspaces(
String schoolDataSource,
List<String> subjects,
List<String> identifiers,
List<SchoolDataIdentifier> educationTypes,
List<SchoolDataIdentifier> curriculumIdentifiers,
String freeText,
List<WorkspaceAccess> accesses,
SchoolDataIdentifier accessUser,
boolean includeUnpublished,
int start,
int maxResults,
List<Sort> sorts) {
if (identifiers != null && identifiers.isEmpty()) {
return new SearchResult(0, 0, 0, new ArrayList<Map<String,Object>>());
}
BoolQueryBuilder query = boolQuery();
freeText = sanitizeSearchString(freeText);
try {
if (!includeUnpublished) {
query.must(termQuery("published", Boolean.TRUE));
}
if (accesses != null) {
BoolQueryBuilder accessQuery = boolQuery();
for (WorkspaceAccess access : accesses) {
switch (access) {
case LOGGED_IN:
case ANYONE:
accessQuery.should(termQuery("access", access));
break;
case MEMBERS_ONLY:
BoolQueryBuilder memberQuery = boolQuery();
IdsQueryBuilder idsQuery = idsQuery("Workspace");
for (SchoolDataIdentifier userWorkspace : getUserWorkspaces(accessUser)) {
idsQuery.addIds(String.format("%s/%s", userWorkspace.getIdentifier(), userWorkspace.getDataSource()));
}
memberQuery.must(idsQuery);
memberQuery.must(termQuery("access", access));
accessQuery.should(memberQuery);
break;
}
}
query.must(accessQuery);
}
if (StringUtils.isNotBlank(schoolDataSource)) {
query.must(termQuery("schoolDataSource", schoolDataSource.toLowerCase()));
}
if (subjects != null && !subjects.isEmpty()) {
query.must(termsQuery("subjectIdentifier", subjects));
}
if (educationTypes != null && !educationTypes.isEmpty()) {
List<String> educationTypeIds = new ArrayList<>(educationTypes.size());
for (SchoolDataIdentifier educationType : educationTypes) {
educationTypeIds.add(educationType.toId());
}
query.must(termsQuery("educationTypeIdentifier.untouched", educationTypeIds));
}
if (!CollectionUtils.isEmpty(curriculumIdentifiers)) {
List<String> curriculumIds = new ArrayList<>(curriculumIdentifiers.size());
for (SchoolDataIdentifier curriculumIdentifier : curriculumIdentifiers) {
curriculumIds.add(curriculumIdentifier.toId());
}
query.must(boolQuery()
.should(termsQuery("curriculumIdentifiers.untouched", curriculumIds))
.should(boolQuery().mustNot(existsQuery("curriculumIdentifiers")))
.minimumNumberShouldMatch(1));
}
if (identifiers != null) {
query.must(termsQuery("identifier", identifiers));
}
if (StringUtils.isNotBlank(freeText)) {
String[] words = freeText.split(" ");
for (int i = 0; i < words.length; i++) {
if (StringUtils.isNotBlank(words[i])) {
query.must(boolQuery()
.should(prefixQuery("name", words[i]))
.should(prefixQuery("description", words[i]))
.should(prefixQuery("subject", words[i]))
.should(prefixQuery("staffMembers.firstName", words[i]))
.should(prefixQuery("staffMembers.lastName", words[i]))
);
}
}
}
SearchRequestBuilder requestBuilder = elasticClient
.prepareSearch("muikku")
.setTypes("Workspace")
.setFrom(start)
.setSize(maxResults);
if (sorts != null && !sorts.isEmpty()) {
for (Sort sort : sorts) {
requestBuilder.addSort(sort.getField(), SortOrder.valueOf(sort.getOrder().name()));
}
}
SearchResponse response = requestBuilder.setQuery(query).execute().actionGet();
List<Map<String, Object>> searchResults = new ArrayList<Map<String, Object>>();
SearchHit[] results = response.getHits().getHits();
for (SearchHit hit : results) {
Map<String, Object> hitSource = hit.getSource();
hitSource.put("indexType", hit.getType());
searchResults.add(hitSource);
}
SearchResult result = new SearchResult(searchResults.size(), start, maxResults, searchResults);
return result;
} catch (Exception e) {
logger.log(Level.SEVERE, "ElasticSearch query failed unexpectedly", e);
return new SearchResult(0, 0, 0, new ArrayList<Map<String,Object>>());
}
}
private Set<SchoolDataIdentifier> getUserWorkspaces(SchoolDataIdentifier userIdentifier) {
Set<SchoolDataIdentifier> result = new HashSet<>();
IdsQueryBuilder query = idsQuery("User");
query.addIds(String.format("%s/%s", userIdentifier.getIdentifier(), userIdentifier.getDataSource()));
SearchResponse response = elasticClient
.prepareSearch("muikku")
.setTypes("User")
.setQuery(query)
.addField("workspaces")
.setSize(1)
.execute()
.actionGet();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
Map<String, SearchHitField> fields = hit.getFields();
SearchHitField workspaceField = fields.get("workspaces");
if (workspaceField != null && workspaceField.getValues() != null) {
for (Object value : workspaceField.getValues()) {
if (value instanceof Number) {
Long workspaceEntityId = ((Number) value).longValue();
WorkspaceEntity workspaceEntity = workspaceEntityController.findWorkspaceEntityById(workspaceEntityId);
if (workspaceEntity != null) {
result.add(new SchoolDataIdentifier(workspaceEntity.getIdentifier(), workspaceEntity.getDataSource().getIdentifier()));
}
}
}
}
}
return result;
}
@Override
public SearchResult searchWorkspaces(String searchTerm, int start, int maxResults) {
return searchWorkspaces(null, null, null, searchTerm, false, start, maxResults);
}
@Override
public SearchResult search(String query, String[] fields, int start, int maxResults, Class<?>... types) {
try {
query = sanitizeSearchString(query);
String[] typenames = new String[types.length];
for (int i = 0; i < types.length; i++) {
typenames[i] = types[i].getSimpleName();
}
SearchRequestBuilder requestBuilder = elasticClient
.prepareSearch("muikku")
.setTypes(typenames)
.setFrom(start)
.setSize(maxResults);
BoolQueryBuilder boolQuery = boolQuery();
for (String field : fields) {
boolQuery.should(prefixQuery(field, query));
}
SearchResponse response = requestBuilder
.setQuery(boolQuery)
.execute()
.actionGet();
List<Map<String, Object>> searchResults = new ArrayList<Map<String, Object>>();
SearchHit[] results = response.getHits().getHits();
for (SearchHit hit : results) {
Map<String, Object> hitSource = hit.getSource();
hitSource.put("indexType", hit.getType());
searchResults.add(hitSource);
}
SearchResult result = new SearchResult(searchResults.size(), start, maxResults, searchResults);
return result;
} catch (Exception e) {
logger.log(Level.SEVERE, "ElasticSearch query failed unexpectedly", e);
return new SearchResult(0, 0, 0, new ArrayList<Map<String,Object>>());
}
}
@Override
public SearchResult freeTextSearch(String text, int start, int maxResults) {
try {
text = sanitizeSearchString(text);
SearchResponse response = elasticClient.prepareSearch().setQuery(matchQuery("_all", text)).setFrom(start).setSize(maxResults).execute()
.actionGet();
List<Map<String, Object>> searchResults = new ArrayList<Map<String, Object>>();
SearchHit[] results = response.getHits().getHits();
for (SearchHit hit : results) {
Map<String, Object> hitSource = hit.getSource();
hitSource.put("indexType", hit.getType());
searchResults.add(hitSource);
}
SearchResult result = new SearchResult(searchResults.size(), start, maxResults, searchResults);
return result;
} catch (Exception e) {
logger.log(Level.SEVERE, "ElasticSearch query failed unexpectedly", e);
return new SearchResult(0, 0, 0, new ArrayList<Map<String,Object>>());
}
}
@Override
public SearchResult matchAllSearch(int start, int maxResults) {
try {
SearchResponse response = elasticClient.prepareSearch().setQuery(matchAllQuery()).setFrom(start).setSize(maxResults).execute().actionGet();
List<Map<String, Object>> searchResults = new ArrayList<Map<String, Object>>();
SearchHit[] results = response.getHits().getHits();
for (SearchHit hit : results) {
Map<String, Object> hitSource = hit.getSource();
hitSource.put("indexType", hit.getType());
searchResults.add(hitSource);
}
SearchResult result = new SearchResult(searchResults.size(), start, maxResults, searchResults);
return result;
} catch (Exception e) {
logger.log(Level.SEVERE, "ElasticSearch query failed unexpectedly", e);
return new SearchResult(0, 0, 0, new ArrayList<Map<String,Object>>());
}
}
@Override
public SearchResult matchAllSearch(int start, int maxResults, Class<?>... types) {
try {
String[] typenames = new String[types.length];
for (int i = 0; i < types.length; i++) {
typenames[i] = types[i].getSimpleName();
}
SearchRequestBuilder requestBuilder = elasticClient
.prepareSearch("muikku")
.setQuery(matchAllQuery())
.setTypes(typenames)
.setFrom(start)
.setSize(maxResults);
SearchResponse response = requestBuilder.execute().actionGet();
List<Map<String, Object>> searchResults = new ArrayList<Map<String, Object>>();
SearchHit[] results = response.getHits().getHits();
for (SearchHit hit : results) {
Map<String, Object> hitSource = hit.getSource();
hitSource.put("indexType", hit.getType());
searchResults.add(hitSource);
}
SearchResult result = new SearchResult(searchResults.size(), start, maxResults, searchResults);
return result;
} catch (Exception e) {
logger.log(Level.SEVERE, "ElasticSearch query failed unexpectedly", e);
return new SearchResult(0, 0, 0, new ArrayList<Map<String,Object>>());
}
}
@Override
public String getName() {
return "elastic-search";
}
private Client elasticClient;
//private Node node;
}