package eu.dnetlib.iis.wf.importer.content;
import static eu.dnetlib.iis.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_CONTENT_MAX_FILE_SIZE_MB;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.io.IOException;
import org.apache.avro.mapred.AvroKey;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import eu.dnetlib.iis.importer.auxiliary.schemas.DocumentContentUrl;
import eu.dnetlib.iis.importer.schemas.DocumentContent;
import eu.dnetlib.iis.wf.importer.content.DocumentContentUrlBasedImporterMapper.InvalidRecordCounters;
/**
* @author mhorst
*
*/
@RunWith(MockitoJUnitRunner.class)
@SuppressWarnings({"rawtypes", "unchecked"})
public class DocumentContentUrlBasedImporterMapperTest {
private DocumentContentUrlBasedImporterMapper mapper;
private byte[] content;
@Mock
private Context context;
@Mock
private Counter sizeExceededCounter;
@Mock
private Counter sizeInvalidCounter;
@Captor
private ArgumentCaptor<AvroKey<DocumentContent>> keyCaptor;
@Captor
private ArgumentCaptor<NullWritable> valueCaptor;
@Before
public void init() throws Exception {
content = "test content".getBytes("utf8");
mapper = new DocumentContentUrlBasedImporterMapper() {
@Override
protected byte[] getContent(String url) throws IOException, InvalidSizeException {
return content;
}
};
doReturn(sizeExceededCounter).when(context).getCounter(InvalidRecordCounters.SIZE_EXCEEDED);
doReturn(sizeInvalidCounter).when(context).getCounter(InvalidRecordCounters.SIZE_INVALID);
}
// --------------------------------- TESTS ---------------------------------
@Test
public void testObtainContent() throws Exception {
// given
Configuration conf = new Configuration();
doReturn(conf).when(context).getConfiguration();
String id = "contentId";
String url = "contentUrl";
DocumentContentUrl docContentUrl = new DocumentContentUrl();
docContentUrl.setId(id);
docContentUrl.setUrl(url);
docContentUrl.setContentSizeKB(1l);
mapper.setup(context);
// execute
mapper.map(new AvroKey<DocumentContentUrl>(docContentUrl), null, context);
// assert
verify(context, times(1)).write(keyCaptor.capture(), valueCaptor.capture());
assertTrue(NullWritable.get() == valueCaptor.getValue());
DocumentContent docContent = keyCaptor.getValue().datum();
assertEquals(id, docContent.getId());
assertEquals(content, docContent.getPdf().array());
verify(sizeExceededCounter, never()).increment(1);
verify(sizeInvalidCounter, never()).increment(1);
}
@Test
public void testContentSizeInvalid() throws Exception {
// given
Configuration conf = new Configuration();
doReturn(conf).when(context).getConfiguration();
String id = "contentId";
String url = "contentUrl";
DocumentContentUrl docContentUrl = new DocumentContentUrl();
docContentUrl.setId(id);
docContentUrl.setUrl(url);
docContentUrl.setContentSizeKB(-1l);
mapper.setup(context);
// execute
mapper.map(new AvroKey<DocumentContentUrl>(docContentUrl), null, context);
// assert
verify(context, never()).write(any(), any());
verify(sizeExceededCounter, never()).increment(1);
verify(sizeInvalidCounter, times(1)).increment(1);
}
@Test
public void testContentSizeInvalidThrown() throws Exception {
// given
mapper = new DocumentContentUrlBasedImporterMapper() {
@Override
protected byte[] getContent(String url) throws IOException, InvalidSizeException {
throw new InvalidSizeException();
}
};
Configuration conf = new Configuration();
doReturn(conf).when(context).getConfiguration();
String id = "contentId";
String url = "contentUrl";
DocumentContentUrl docContentUrl = new DocumentContentUrl();
docContentUrl.setId(id);
docContentUrl.setUrl(url);
docContentUrl.setContentSizeKB(1l);
mapper.setup(context);
// execute
mapper.map(new AvroKey<DocumentContentUrl>(docContentUrl), null, context);
// assert
verify(context, never()).write(any(), any());
verify(sizeExceededCounter, never()).increment(1);
verify(sizeInvalidCounter, times(1)).increment(1);
}
@Test(expected=IOException.class)
public void testIOExceptionThrown() throws Exception {
// given
mapper = new DocumentContentUrlBasedImporterMapper() {
@Override
protected byte[] getContent(String url) throws IOException, InvalidSizeException {
throw new IOException();
}
};
Configuration conf = new Configuration();
doReturn(conf).when(context).getConfiguration();
String id = "contentId";
String url = "contentUrl";
DocumentContentUrl docContentUrl = new DocumentContentUrl();
docContentUrl.setId(id);
docContentUrl.setUrl(url);
docContentUrl.setContentSizeKB(1l);
mapper.setup(context);
// execute
mapper.map(new AvroKey<DocumentContentUrl>(docContentUrl), null, context);
}
@Test
public void testContentSizeExceeded() throws Exception {
// given
Configuration conf = new Configuration();
conf.set(IMPORT_CONTENT_MAX_FILE_SIZE_MB, "1");
doReturn(conf).when(context).getConfiguration();
String id = "contentId";
String url = "contentUrl";
DocumentContentUrl docContentUrl = new DocumentContentUrl();
docContentUrl.setId(id);
docContentUrl.setUrl(url);
docContentUrl.setContentSizeKB(1025l);
mapper.setup(context);
// execute
mapper.map(new AvroKey<DocumentContentUrl>(docContentUrl), null, context);
// assert
verify(context, never()).write(any(), any());
verify(sizeExceededCounter, times(1)).increment(1);
verify(sizeInvalidCounter, never()).increment(1);
}
}