package io.monokkel.core;
import com.google.common.collect.Maps;
import io.monokkel.domain.PageData;
import io.monokkel.exceptions.IndexDocumentException;
import io.monokkel.factories.ClientFactory;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.*;
public class IndexerTest {
@Test
public void indexParserOutput_withOneEntry_expectPrepareIndexToBeCalledWithParameters() throws IOException, IndexDocumentException {
final String indexType = "snapshot";
final String indexName = "finn.no";
final PageData pageData = createResponseOutput("http://url", 100L, "<html>", null, null);
final ClientFactory clientFactory = mock(ClientFactory.class);
final TransportClient mockedClient = mock(TransportClient.class);
setupMocks(indexType, indexName, clientFactory, mockedClient, mock(IndexRequestBuilder.class));
Indexer indexerTask = new Indexer(indexName, indexType, clientFactory, true);
indexerTask.indexParserOutput(pageData);
verify(mockedClient, Mockito.times(1)).prepareIndex(eq(indexName),eq(indexType));
}
@Test
public void indexParserOutput_withOneEntry_expectJsonToBeWritten() throws IOException, IndexDocumentException {
final PageData pageData = createResponseOutput("http://url", 100L, "<html>extracted</html>", "extracted", null);
final ClientFactory clientFactory = mock(ClientFactory.class);
final IndexRequestBuilder mockedIndex = mock(IndexRequestBuilder.class);
setupMocks(clientFactory, mockedIndex);
Indexer indexerTask = createIndexer(clientFactory);
indexerTask.indexParserOutput(pageData);
verify(mockedIndex,times(1)).setSource("{\"page\":{\"title\":null,\"url\":\"http://url\",\"date\":\"1970-01-01T00:00:00.100Z\",\"content\":\"extracted\",\"transformed_content\":null,\"raw_response\":\"<html>extracted</html>\"}}");
}
@Test
public void indexParserOutput_withOneEntryAndNoTimeStampInId_expectJsonToBeWritten() throws IOException, IndexDocumentException {
final PageData pageData = createResponseOutput("http://url", 100L, "<html>extracted</html>", "extracted", null);
final ClientFactory clientFactory = mock(ClientFactory.class);
final IndexRequestBuilder mockedIndex = mock(IndexRequestBuilder.class);
setupMocks(clientFactory, mockedIndex);
Indexer indexerTask = createIndexer(clientFactory);
indexerTask.indexParserOutput(pageData);
verify(mockedIndex,times(1)).setSource("{\"page\":{\"title\":null,\"url\":\"http://url\",\"date\":\"1970-01-01T00:00:00.100Z\",\"content\":\"extracted\",\"transformed_content\":null,\"raw_response\":\"<html>extracted</html>\"}}");
}
@Test
public void indexParserOutput_withOneEntryInTheQueue_expectIndexerCommandToBeExecuted() throws IOException, IndexDocumentException {
final PageData pageData = createResponseOutput("http://url", 100L, "<html>", null, null);
final ClientFactory clientFactory = mock(ClientFactory.class);
final IndexRequestBuilder mockedIndex = mock(IndexRequestBuilder.class);
setupMocks(clientFactory, mockedIndex);
Indexer indexerTask = createIndexer(clientFactory);
indexerTask.indexParserOutput(pageData);
verify(mockedIndex,times(1)).execute();
}
@Test
public void indexParserOutput_withOneEntry_expectJsonToBeWrittenWithTransformation() throws IOException, IndexDocumentException {
final HashMap<String, String> transformationMap = Maps.newHashMap();
transformationMap.put("content","output");
final PageData pageData = createResponseOutput("http://url", 100L, "<html>extracted</html>", "extracted", transformationMap);
final ClientFactory clientFactory = mock(ClientFactory.class);
final IndexRequestBuilder mockedIndex = mock(IndexRequestBuilder.class);
setupMocks(clientFactory, mockedIndex);
Indexer indexerTask = createIndexer(clientFactory);
indexerTask.indexParserOutput(pageData);
verify(mockedIndex,times(1)).setSource("{\"page\":{\"title\":null,\"url\":\"http://url\",\"date\":\"1970-01-01T00:00:00.100Z\",\"content\":\"extracted\",\"transformed_content\":{\"content\":\"output\"},\"raw_response\":\"<html>extracted</html>\"}}");
}
@Test
public void indexParserOutput_withOneEntryAndIndexRawFalse_expectJsonToBeWrittenWithTransformation() throws IOException, IndexDocumentException {
final HashMap<String, String> transformationMap = Maps.newHashMap();
transformationMap.put("content","output");
final PageData pageData = createResponseOutput("http://url", 100L, "<html>extracted</html>", "extracted", transformationMap);
final ClientFactory clientFactory = mock(ClientFactory.class);
final IndexRequestBuilder mockedIndex = mock(IndexRequestBuilder.class);
setupMocks(clientFactory, mockedIndex);
Indexer indexerTask = createIndexer(clientFactory,false);
indexerTask.indexParserOutput(pageData);
verify(mockedIndex,times(1)).setSource("{\"page\":{\"title\":null,\"url\":\"http://url\",\"date\":\"1970-01-01T00:00:00.100Z\",\"content\":\"extracted\",\"transformed_content\":{\"content\":\"output\"}}}");
}
private void setupMocks(ClientFactory clientFactory, IndexRequestBuilder mockedIndex) {
setupMocks("snapshot", "finn.no", clientFactory, mock(TransportClient.class), mockedIndex);
}
private Indexer createIndexer(ClientFactory clientFactory,Boolean shouldIndexRaw) {
return new Indexer("finn.no", "snapshot", clientFactory, shouldIndexRaw);
}
private Indexer createIndexer(ClientFactory clientFactory) {
return createIndexer(clientFactory,true);
}
@Test
public void close_withInitializedClient_expectClientClosed() throws IOException, IndexDocumentException {
final ClientFactory clientFactory = mock(ClientFactory.class);
final TransportClient mockedClient = mock(TransportClient.class);
setupMocks("snapshot", "finn.no", clientFactory, mockedClient, mock(IndexRequestBuilder.class));
Indexer indexerTask = createIndexer(clientFactory);
indexerTask.close();
verify(mockedClient, times(1)).close();
}
@Test
public void close_withNotInitializedClient_expectNothingToBeCalledAndNoExceptionsThrown() throws IOException, IndexDocumentException {
final ClientFactory clientFactory = mock(ClientFactory.class);
Mockito.when(clientFactory.buildStandardClientConnectedToOneNode()).thenReturn(null);
Indexer indexerTask = createIndexer(clientFactory);
indexerTask.close();
}
private PageData createResponseOutput(final String url, final long timestamp, final String content, final String extractedContent, final Map<String,String> transformationMap) {
final PageData pageData = new PageData(null,url,timestamp,content,null, extractedContent,transformationMap);
return pageData;
}
private void setupMocks(final String indexType, final String indexName, final ClientFactory clientFactory, final TransportClient mockedClient, final IndexRequestBuilder mockedIndex) {
Mockito.when(clientFactory.buildStandardClientConnectedToOneNode()).thenReturn(mockedClient);
Mockito.when(mockedClient.prepareIndex(eq(indexName), eq(indexType))).thenReturn(mockedIndex);
}
}