/**
*
* Copyright
* 2009-2015 Jayway Products AB
* 2016-2017 Föreningen Sambruk
*
* Licensed under AGPL, Version 3.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.gnu.org/licenses/agpl.txt
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package se.streamsource.infrastructure.index.elasticsearch;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.qi4j.api.configuration.Configuration;
import org.qi4j.api.entity.EntityReference;
import org.qi4j.api.injection.scope.Service;
import org.qi4j.api.injection.scope.Structure;
import org.qi4j.api.injection.scope.This;
import org.qi4j.api.mixin.Mixins;
import org.qi4j.api.service.qualifier.Tagged;
import org.qi4j.api.structure.Module;
import org.qi4j.api.usecase.UsecaseBuilder;
import org.qi4j.api.util.Classes;
import org.qi4j.api.util.Function;
import org.qi4j.api.util.Iterables;
import org.qi4j.spi.entity.*;
import org.qi4j.spi.entity.association.AssociationDescriptor;
import org.qi4j.spi.entity.association.ManyAssociationDescriptor;
import org.qi4j.spi.entitystore.EntityStore;
import org.qi4j.spi.entitystore.EntityStoreUnitOfWork;
import org.qi4j.spi.entitystore.StateChangeListener;
import org.qi4j.spi.property.PropertyDescriptor;
import org.qi4j.spi.property.PropertyType;
import org.qi4j.spi.property.ValueType;
import org.qi4j.spi.structure.ModuleSPI;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import se.streamsource.streamflow.util.Primitives;
import java.lang.reflect.*;
import java.util.*;
/**
* Back ported from Qi4j 2.0
* courtesy of Paul Merlin
*
* Listen to Entity state changes and index them in ElasticSearch.
*
* QUID Use two indices, one for strict queries, one for full text and fuzzy search?
*/
@Mixins( ElasticSearchIndexer.Mixin.class )
public interface ElasticSearchIndexer
extends StateChangeListener
{
class Mixin
implements StateChangeListener
{
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchIndexer.class);
@Structure
private ModuleSPI module;
@Service
private EntityStore entityStore;
@This
private ElasticSearchSupport support;
@This
Configuration<ElasticSearchConfiguration> config;
public void emptyIndex()
{
support.client().admin().indices().prepareDelete( support.index() ).execute().actionGet();
}
public void notifyChanges( Iterable<EntityState> changedStates )
{
long start1 = System.nanoTime();
// All updated or new states
Map<String, EntityState> newStates = new HashMap<String, EntityState>();
for( EntityState eState : changedStates )
{
if( eState.status() == EntityStatus.UPDATED || eState.status() == EntityStatus.NEW )
{
newStates.put( eState.identity().identity(), eState );
}
}
EntityStoreUnitOfWork uow = entityStore.newUnitOfWork( UsecaseBuilder.newUsecase("Load associations for indexing"),
module );
// Bulk index request builder
BulkRequestBuilder bulkBuilder = support.client().prepareBulk();
// Handle changed entity states
for( EntityState changedState : changedStates )
{
if( changedState.entityDescriptor().entityType().queryable() )
{
switch( changedState.status() )
{
case REMOVED:
LOGGER.trace( "Removing Entity State from Index: {}", changedState );
remove( bulkBuilder, changedState.identity().identity() );
break;
case UPDATED:
LOGGER.trace( "Updating Entity State in Index: {}", changedState );
remove( bulkBuilder, changedState.identity().identity() );
String updatedJson = toJSON( changedState, newStates, uow );
LOGGER.trace( "Will index: {}", updatedJson );
index( bulkBuilder, changedState.identity().identity(), updatedJson );
break;
case NEW:
LOGGER.trace( "Creating Entity State in Index: {}", changedState );
String newJson = toJSON( changedState, newStates, uow );
LOGGER.trace( "Will index: {}", newJson );
index( bulkBuilder, changedState.identity().identity(), newJson );
break;
case LOADED:
default:
// Ignored
break;
}
}
}
uow.discard();
long end1 = System.nanoTime();
long timeMicro1 = (end1 - start1) / 1000;
double timeMilli1 = timeMicro1 / 1000.0;
LOGGER.trace( "NotifyChanges first part took {}ms", timeMilli1 );
if( bulkBuilder.numberOfActions() > 0 )
{
try
{
// Execute bulk actions
BulkResponse bulkResponse = bulkBuilder.execute().actionGet();
// Handle errors
if( bulkResponse.hasFailures() )
{
throw new ElasticSearchIndexException( bulkResponse.buildFailureMessage() );
}
LOGGER.trace( "Indexing changed Entity states took {}ms", bulkResponse.getTookInMillis() );
if( config.configuration().indexRefreshInterval().get() == null
|| "-1".equals( config.configuration().indexRefreshInterval().get() ) )
{
// Refresh index manually if automatic is switched off
long start2 = System.nanoTime();
support.client().admin().indices().prepareRefresh( support.index() ).execute().actionGet();
long end2 = System.nanoTime();
long timeMicro2 = (end2 - start2) / 1000;
double timeMilli2 = timeMicro2 / 1000.0;
LOGGER.trace( "Indexing refresh index took {}ms", timeMilli2 );
}
} catch( ElasticsearchIllegalStateException esis )
{
LOGGER.warn( "Possible corrupted index state.", esis );
if(!config.configuration().suppressInterruptedException().get() )
{
throw esis;
}
}
}
}
private void remove( BulkRequestBuilder bulkBuilder, String identity )
{
bulkBuilder.add( support.client().
prepareDelete( support.index(), support.entitiesType(), identity ) );
}
private void index( BulkRequestBuilder bulkBuilder, String identity, String json )
{
bulkBuilder.add( support.client().
prepareIndex( support.index(), support.entitiesType(), identity ).
setSource( json ) );
}
/**
* <pre>
* {
* "_identity": "ENTITY-IDENTITY",
* "_types": [ "All", "Entity", "types" ],
* "property.name": property.value,
* "association.name": "ASSOCIATED-IDENTITY",
* "manyassociation.name": [ "ASSOCIATED", "IDENTITIES" ]
* }
* </pre>
*/
private String toJSON( EntityState state, Map<String, EntityState> newStates, EntityStoreUnitOfWork uow )
{
long start = System.nanoTime();
JSONObject json = null;
try
{
json = new JSONObject();
json.put( "_identity", state.identity().identity() );
json.put( "_types", Iterables.addAll( new ArrayList<String>(), Iterables.map( toClassName(), state.entityDescriptor().mixinTypes()) ) );
EntityType entityType = state.entityDescriptor().entityType();
EntityDescriptor entityDesc = state.entityDescriptor();
// Properties
for( PropertyType propType : entityType.properties() )
{
if( propType.queryable() )
{
String key = propType.qualifiedName().name();
Object value = state.getProperty(propType.qualifiedName());
if( value == null || Primitives.isPrimitiveValue(value) )
{
json.put( key, value );
}
else
{
// TODO Theses tests are pretty fragile, find a better way to fix this, Jackson API should behave better
String serialized = propType.type().toJSON(value).toString();
if( serialized.startsWith( "{" ) )
{
json.put( key, new JSONObject( serialized ) );
}
else if( serialized.startsWith( "[" ) )
{
json.put( key, new JSONArray( serialized ) );
}
else
{
json.put( key, serialized );
}
}
}
}
// Associations
for( AssociationDescriptor assocDesc : entityDesc.state().associations() )
{
if( assocDesc.associationType().queryable() )
{
String key = assocDesc.qualifiedName().name();
EntityReference associated = state.getAssociation(assocDesc.qualifiedName());
Object value;
if( associated == null )
{
value = null;
}
else
{
if( assocDesc.isAggregated() || support.indexNonAggregatedAssociations() )
{
if( newStates.containsKey( associated.identity() ) )
{
value = new JSONObject( toJSON( newStates.get( associated.identity() ), newStates, uow ) );
}
else
{
EntityState assocState = uow.getEntityState( EntityReference.parseEntityReference( associated.identity() ) );
value = new JSONObject( toJSON( assocState, newStates, uow ) );
}
}
else
{
value = new JSONObject( Collections.singletonMap("identity", associated.identity()) );
}
}
json.put( key, value );
}
}
// ManyAssociations
for( ManyAssociationDescriptor manyAssocDesc : entityDesc.state().manyAssociations() )
{
if( manyAssocDesc.manyAssociationType().queryable() )
{
String key = manyAssocDesc.qualifiedName().name();
JSONArray array = new JSONArray();
ManyAssociationState associateds = state.getManyAssociation(manyAssocDesc.qualifiedName());
for( EntityReference associated : associateds )
{
if( manyAssocDesc.isAggregated() || support.indexNonAggregatedAssociations() )
{
if( newStates.containsKey( associated.identity() ) )
{
array.put( new JSONObject( toJSON( newStates.get( associated.identity() ), newStates, uow ) ) );
}
else
{
EntityState assocState = uow.getEntityState(EntityReference.parseEntityReference(associated.identity()));
array.put( new JSONObject( toJSON( assocState, newStates, uow ) ) );
}
}
else
{
array.put( new JSONObject( Collections.singletonMap( "identity", associated.identity() ) ) );
}
}
json.put( key, array );
}
}
return json.toString();
}
catch( JSONException e )
{
throw new ElasticSearchIndexException( "Could not index EntityState", e );
}
}
private Function<Type, String> toClassName()
{
return new Function<Type, String>()
{
public String map( Type type )
{
return RAW_CLASS.map( type ).getName();
}
};
}
/**
* Function that extract the raw class of a type.
*/
private final Function<Type, Class<?>> RAW_CLASS = new Function<Type, Class<?>>()
{
public Class<?> map( Type genericType )
{
// Calculate raw type
if( genericType instanceof Class )
{
return (Class<?>) genericType;
}
else if( genericType instanceof ParameterizedType)
{
return (Class<?>) ( (ParameterizedType) genericType ).getRawType();
}
else if( genericType instanceof TypeVariable)
{
return (Class<?>) ( (TypeVariable) genericType ).getGenericDeclaration();
}
else if( genericType instanceof WildcardType)
{
return (Class<?>) ( (WildcardType) genericType ).getUpperBounds()[ 0 ];
}
else if( genericType instanceof GenericArrayType)
{
Object temp = Array.newInstance( (Class<?>) ( (GenericArrayType) genericType ).getGenericComponentType(), 0 );
return temp.getClass();
}
throw new IllegalArgumentException( "Could not extract the raw class of " + genericType );
}
};
}
}