package org.molgenis.data.elasticsearch.index.job;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.molgenis.data.DataService;
import org.molgenis.data.Entity;
import org.molgenis.data.MolgenisDataException;
import org.molgenis.data.Query;
import org.molgenis.data.elasticsearch.ElasticsearchService.IndexingMode;
import org.molgenis.data.elasticsearch.SearchService;
import org.molgenis.data.index.meta.*;
import org.molgenis.data.jobs.Progress;
import org.molgenis.data.meta.MetaDataService;
import org.molgenis.data.meta.model.EntityType;
import org.molgenis.data.support.QueryImpl;
import org.molgenis.test.data.AbstractMolgenisSpringTest;
import org.molgenis.test.data.EntityTestHarness;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.core.Authentication;
import org.springframework.test.context.ContextConfiguration;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.stream.Stream;
import static com.google.common.collect.Lists.newArrayList;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Stream.empty;
import static java.util.stream.Stream.of;
import static org.mockito.Mockito.*;
import static org.mockito.MockitoAnnotations.initMocks;
import static org.molgenis.data.index.meta.IndexActionGroupMetaData.INDEX_ACTION_GROUP;
import static org.molgenis.data.index.meta.IndexActionMetaData.INDEX_ACTION;
import static org.molgenis.data.index.meta.IndexActionMetaData.IndexStatus.FAILED;
import static org.molgenis.data.index.meta.IndexActionMetaData.IndexStatus.FINISHED;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertSame;
@ContextConfiguration(classes = { IndexJobTest.Config.class })
public class IndexJobTest extends AbstractMolgenisSpringTest
{
@Captor
private ArgumentCaptor<Stream<Entity>> streamCaptor;
@Autowired
private Progress progress;
@Autowired
private Authentication authentication;
@Autowired
private SearchService searchService;
@Autowired
private MetaDataService mds;
@Autowired
private Config config;
@Autowired
private DataService dataService;
@Autowired
private EntityTestHarness harness;
@Autowired
private IndexActionFactory indexActionFactory;
@Autowired
private IndexActionGroupFactory indexActionGroupFactory;
private final String transactionId = "aabbcc";
private IndexJob indexJob;
private IndexActionGroup indexActionGroup;
private EntityType testEntityType;
private Entity toIndexEntity;
@BeforeMethod
public void beforeMethod()
{
initMocks(this);
config.resetMocks();
indexJob = new IndexJob(progress, authentication, transactionId, dataService, searchService);
indexActionGroup = indexActionGroupFactory.create(transactionId).setCount(0);
when(dataService.findOneById(INDEX_ACTION_GROUP, transactionId, IndexActionGroup.class))
.thenReturn(indexActionGroup);
when(dataService.getMeta()).thenReturn(mds);
testEntityType = harness.createDynamicRefEntityType();
when(mds.getEntityType("test")).thenReturn(testEntityType);
toIndexEntity = harness.createTestRefEntities(testEntityType, 1).get(0);
when(dataService.getEntityType("test")).thenReturn(testEntityType);
when(dataService.findOneById("test", "entityId")).thenReturn(toIndexEntity);
}
@Test
public void testNoIndexActionJobForTransaction()
{
when(dataService.findOneById(INDEX_ACTION_GROUP, this.transactionId)).thenReturn(null);
mockGetAllIndexActions(empty());
indexJob.call(this.progress);
verify(progress).status("No index actions found for transaction id: [aabbcc]");
verify(searchService, never()).refreshIndex();
}
@Test
public void testNoIndexActionsForTransaction()
{
mockGetAllIndexActions(empty());
indexJob.call(this.progress);
verify(progress).status("No index actions found for transaction id: [aabbcc]");
verify(searchService, never()).refreshIndex();
}
private void mockGetAllIndexActions(Stream<IndexAction> entities)
{
Query<IndexAction> q = IndexJob.createQueryGetAllIndexActions(transactionId);
when(dataService.findAll(INDEX_ACTION, q, IndexAction.class)).thenReturn(entities);
}
@Test
public void testCreateQueryGetAllIndexActions()
{
Query<IndexAction> q = IndexJob.createQueryGetAllIndexActions("testme");
assertEquals(q.toString(),
"rules=['indexActionGroup' = 'testme'], sort=Sort [orders=[Order [attr=actionOrder, direction=ASC]]]");
}
@Test
public void rebuildIndexDeleteSingleEntityTest()
{
when(dataService.findOneById("test", "entityId")).thenReturn(null);
IndexAction indexAction = indexActionFactory.create().setIndexActionGroup(indexActionGroup)
.setEntityFullName("test").setEntityId("entityId").setActionOrder(0)
.setIndexStatus(IndexActionMetaData.IndexStatus.PENDING);
mockGetAllIndexActions(of(indexAction));
indexActionGroup.setCount(1);
when(dataService.hasRepository("test")).thenReturn(true);
indexJob.call(progress);
assertEquals(indexAction.getIndexStatus(), FINISHED);
verify(searchService).deleteById("entityId", testEntityType);
// verify progress messages
verify(progress).status("Start indexing for transaction id: [aabbcc]");
verify(progress).setProgressMax(1);
verify(progress).progress(0, "Indexing test.entityId");
verify(progress).progress(1, "Executed all index actions, cleaning up the actions...");
verify(progress).status("Refresh index start");
verify(progress).status("Refresh index done");
verify(progress).status("Finished indexing for transaction id: [aabbcc]");
verify(searchService).refreshIndex();
verify(dataService, times(2)).update(INDEX_ACTION, indexAction);
}
@Test
public void rebuildIndexCreateSingleEntityTest()
{
this.rebuildIndexSingleEntityTest(IndexingMode.ADD);
}
@Test
public void rebuildIndexUpdateSingleEntityTest()
{
Entity actualEntity = dataService.findOneById("test", "entityId");
EntityType emd = actualEntity.getEntityType();
Query<Entity> q = new QueryImpl<>();
q.eq(emd.getIdAttribute().getName(), "entityId");
when(searchService.findOne(q, emd)).thenReturn(actualEntity);
this.rebuildIndexSingleEntityTest(IndexingMode.UPDATE);
}
private void rebuildIndexSingleEntityTest(IndexingMode indexingMode)
{
IndexAction indexAction = indexActionFactory.create().setIndexActionGroup(indexActionGroup)
.setEntityFullName("test").setEntityId("entityId").setActionOrder(0)
.setIndexStatus(IndexActionMetaData.IndexStatus.PENDING);
mockGetAllIndexActions(of(indexAction));
when(dataService.hasRepository("test")).thenReturn(true);
indexActionGroup.setCount(1);
indexJob.call(this.progress);
assertEquals(indexAction.getIndexStatus(), FINISHED);
verify(this.searchService).index(toIndexEntity, testEntityType, indexingMode);
verify(progress).status("Start indexing for transaction id: [aabbcc]");
verify(progress).setProgressMax(1);
verify(progress).progress(0, "Indexing test.entityId");
verify(progress).progress(1, "Executed all index actions, cleaning up the actions...");
verify(progress).status("Refresh index start");
verify(progress).status("Refresh index done");
verify(progress).status("Finished indexing for transaction id: [aabbcc]");
verify(dataService, times(2)).update(INDEX_ACTION, indexAction);
}
@Test
private void rebuildIndexMetaUpdateDataTest()
{
when(dataService.hasRepository("test")).thenReturn(true);
EntityType entityType = dataService.getEntityType("test");
when(searchService.hasMapping(entityType)).thenReturn(true);
IndexAction indexAction = indexActionFactory.create().setIndexActionGroup(indexActionGroup)
.setEntityFullName("test").setEntityId(null).setActionOrder(0)
.setIndexStatus(IndexActionMetaData.IndexStatus.PENDING);
mockGetAllIndexActions(of(indexAction));
indexActionGroup.setCount(1);
indexJob.call(this.progress);
assertEquals(indexAction.getIndexStatus(), FINISHED);
verify(this.searchService).rebuildIndex(this.dataService.getRepository("any"));
verify(progress).status("Start indexing for transaction id: [aabbcc]");
verify(progress).setProgressMax(1);
verify(progress).progress(0, "Indexing test");
verify(progress).progress(1, "Executed all index actions, cleaning up the actions...");
verify(progress).status("Refresh index start");
verify(progress).status("Refresh index done");
verify(progress).status("Finished indexing for transaction id: [aabbcc]");
verify(dataService, times(2)).update(INDEX_ACTION, indexAction);
// make sure both the actions and the action job got deleted
verify(dataService).delete(eq(INDEX_ACTION), streamCaptor.capture());
assertEquals(streamCaptor.getValue().collect(toList()), newArrayList(indexAction));
verify(dataService).deleteById(INDEX_ACTION_GROUP, transactionId);
}
@Test
private void rebuildIndexMetaCreateDataTest()
{
when(dataService.hasRepository("test")).thenReturn(true);
IndexAction indexAction = indexActionFactory.create().setIndexActionGroup(indexActionGroup)
.setEntityFullName("test").setEntityId(null).setActionOrder(0)
.setIndexStatus(IndexActionMetaData.IndexStatus.PENDING);
mockGetAllIndexActions(of(indexAction));
indexActionGroup.setCount(1);
indexJob.call(this.progress);
assertEquals(indexAction.getIndexStatus(), FINISHED);
verify(this.searchService).rebuildIndex(this.dataService.getRepository("any"));
verify(progress).status("Start indexing for transaction id: [aabbcc]");
verify(progress).setProgressMax(1);
verify(progress).progress(0, "Indexing test");
verify(progress).progress(1, "Executed all index actions, cleaning up the actions...");
verify(progress).status("Refresh index start");
verify(progress).status("Refresh index done");
verify(progress).status("Finished indexing for transaction id: [aabbcc]");
verify(dataService, times(2)).update(INDEX_ACTION, indexAction);
// make sure both the actions and the action job got deleted
verify(dataService).delete(eq(INDEX_ACTION), streamCaptor.capture());
assertEquals(streamCaptor.getValue().collect(toList()), newArrayList(indexAction));
verify(dataService).deleteById(INDEX_ACTION_GROUP, transactionId);
verify(dataService).deleteById(INDEX_ACTION_GROUP, transactionId);
}
@Test
public void rebuildIndexDeleteMetaDataEntityTest()
{
IndexAction indexAction = indexActionFactory.create().setIndexActionGroup(indexActionGroup)
.setEntityFullName("test").setEntityId(null).setActionOrder(0)
.setIndexStatus(IndexActionMetaData.IndexStatus.PENDING);
mockGetAllIndexActions(of(indexAction));
indexActionGroup.setCount(1);
when(dataService.hasRepository("test")).thenReturn(false);
when(searchService.hasMapping("test")).thenReturn(true);
indexJob.call(this.progress);
assertEquals(indexAction.getIndexStatus(), FINISHED);
verify(this.searchService).delete("test");
verify(progress).status("Start indexing for transaction id: [aabbcc]");
verify(progress).setProgressMax(1);
verify(progress).progress(0, "Dropping test");
verify(progress).progress(1, "Executed all index actions, cleaning up the actions...");
verify(progress).status("Refresh index start");
verify(progress).status("Refresh index done");
verify(progress).status("Finished indexing for transaction id: [aabbcc]");
verify(dataService, times(2)).update(INDEX_ACTION, indexAction);
}
@Test
public void indexSingleEntitySearchServiceThrowsExceptionOnSecondEntityId()
{
IndexAction indexAction1 = indexActionFactory.create().setIndexActionGroup(indexActionGroup)
.setEntityFullName("test").setEntityId("entityId1").setActionOrder(0)
.setIndexStatus(IndexActionMetaData.IndexStatus.PENDING);
IndexAction indexAction2 = indexActionFactory.create().setIndexActionGroup(indexActionGroup)
.setEntityFullName("test").setEntityId("entityId2").setActionOrder(1)
.setIndexStatus(IndexActionMetaData.IndexStatus.PENDING);
IndexAction indexAction3 = indexActionFactory.create().setIndexActionGroup(indexActionGroup)
.setEntityFullName("test").setEntityId("entityId3").setActionOrder(2)
.setIndexStatus(IndexActionMetaData.IndexStatus.PENDING);
mockGetAllIndexActions(of(indexAction1, indexAction2, indexAction3));
indexActionGroup.setCount(3);
MolgenisDataException mde = new MolgenisDataException("Random unrecoverable exception");
doThrow(mde).when(searchService).deleteById("entityId2", testEntityType);
when(dataService.hasRepository("test")).thenReturn(true);
try
{
indexJob.call(progress);
}
catch (Exception expected)
{
assertSame(expected, mde);
}
verify(searchService).deleteById("entityId1", testEntityType);
verify(searchService).deleteById("entityId2", testEntityType);
verify(searchService).deleteById("entityId3", testEntityType);
verify(searchService).refreshIndex();
// Make sure the action status got updated and that the actionJob didn't get deleted
assertEquals(indexAction1.getIndexStatus(), FINISHED);
assertEquals(indexAction2.getIndexStatus(), FAILED);
verify(dataService, atLeast(1)).update(INDEX_ACTION, indexAction1);
verify(dataService, atLeast(1)).update(INDEX_ACTION, indexAction2);
verify(dataService, never()).delete(INDEX_ACTION_GROUP, indexActionGroup);
}
@Configuration
@ComponentScan({ "org.molgenis.data.index", "org.molgenis.test.data" })
public static class Config
{
@Mock
private Progress progress;
@Mock
private Authentication authentication;
@Mock
private SearchService searchService;
@Mock
private MetaDataService mds;
@Mock
private DataService dataService;
public Config()
{
initMocks(this);
}
@Bean
public Progress progress()
{
return progress;
}
@Bean
public Authentication authentication()
{
return authentication;
}
@Bean
public SearchService searchService()
{
return searchService;
}
@Bean
public MetaDataService metaDataService()
{
return mds;
}
@Bean
public DataService dataService()
{
return dataService;
}
void resetMocks()
{
reset(progress, authentication, searchService, mds, dataService);
}
}
}