package dk.kb.yggdrasil.preservation; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.stub; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; import java.io.File; import java.util.Arrays; import java.util.Date; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import dk.kb.yggdrasil.HttpCommunication; import dk.kb.yggdrasil.HttpPayload; import dk.kb.yggdrasil.bitmag.Bitrepository; import dk.kb.yggdrasil.config.Models; import dk.kb.yggdrasil.config.RequestHandlerContext; import dk.kb.yggdrasil.config.YggdrasilConfig; import dk.kb.yggdrasil.db.PreservationRequestState; import dk.kb.yggdrasil.db.StateDatabase; import dk.kb.yggdrasil.exceptions.YggdrasilException; import dk.kb.yggdrasil.json.preservation.PreservationRequest; import dk.kb.yggdrasil.messaging.RemotePreservationStateUpdater; import dk.kb.yggdrasil.testutils.MetadataContentUtils; @RunWith(JUnit4.class) public class PreservationRequestHandlerTest { protected static final String NON_RANDOM_UUID = "random-uuid"; protected static final String NON_RANDOM_FILE_UUID = "random-file-uuid"; protected static final String DEFAULT_COLLECTION = "collection"; protected static File generalConfigFile = new File("src/test/resources/config/yggdrasil.yml"); protected static File modelsFile = new File("src/test/resources/config/models.yml"); protected static File testFileDir = new File("temporarydir"); protected static PreservationRequest request; protected static YggdrasilConfig config; protected static Models models; @BeforeClass public static void beforeClass() throws Exception { System.setProperty("dk.kb.yggdrasil.runningmode", "test"); config = new YggdrasilConfig(generalConfigFile); models = new Models(modelsFile); request = makeRequest(); } @Test public void testSuccessCase() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(true); RequestHandlerContext context = new RequestHandlerContext(bitrepository, config, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); prh.handleRequest(request); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_REQUEST_RECEIVED)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_METADATA_PACKAGED_SUCCESSFULLY)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_COMPLETE)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_WAITING_FOR_MORE_DATA)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_UPLOAD_SUCCESS)); verifyNoMoreInteractions(updater); verify(states, times(3)).putPreservationRecord(eq(NON_RANDOM_UUID), any(PreservationRequestState.class)); verify(states).delete(eq(NON_RANDOM_UUID)); verifyNoMoreInteractions(states); verify(bitrepository).getKnownCollections(); verify(bitrepository).uploadFile(any(File.class), eq(DEFAULT_COLLECTION)); verifyZeroInteractions(httpCommunication); } @Test public void testInvalidRequest() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(true); RequestHandlerContext context = new RequestHandlerContext(bitrepository, config, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); PreservationRequest badRequest = new PreservationRequest(); prh.handleRequest(badRequest); verifyZeroInteractions(updater); verifyZeroInteractions(bitrepository); verifyZeroInteractions(states); verifyZeroInteractions(httpCommunication); } @Test public void testInvalidModelInRequest() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(true); RequestHandlerContext context = new RequestHandlerContext(bitrepository, config, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); PreservationRequest badRequest = makeRequest(); badRequest.Model = "ThisIsDefinitelyNotAValidModel"; try { prh.handleRequest(badRequest); Assert.fail("Should throw an exception"); } catch (YggdrasilException e) { // expected } verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_REQUEST_RECEIVED)); verify(updater).sendPreservationResponseWithSpecificDetails(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_REQUEST_FAILED), anyString()); verifyNoMoreInteractions(updater); verify(bitrepository).getKnownCollections(); verifyNoMoreInteractions(bitrepository); verify(states).putPreservationRecord(eq(NON_RANDOM_UUID), any(PreservationRequestState.class)); verify(states).delete(eq(NON_RANDOM_UUID)); verifyNoMoreInteractions(states); verifyZeroInteractions(httpCommunication); } @Test public void testInvalidMetadataInRequest() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(true); RequestHandlerContext context = new RequestHandlerContext(bitrepository, config, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); PreservationRequest badRequest = makeRequest(); badRequest.metadata = "<metadata />"; try { prh.handleRequest(badRequest); Assert.fail("Should throw an exception"); } catch (YggdrasilException e) { // expected } verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_REQUEST_RECEIVED)); verify(updater).sendPreservationResponseWithSpecificDetails(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_METADATA_PACKAGED_FAILURE), anyString()); verifyNoMoreInteractions(updater); verify(bitrepository).getKnownCollections(); verifyNoMoreInteractions(bitrepository); verify(states).putPreservationRecord(eq(NON_RANDOM_UUID), any(PreservationRequestState.class)); verify(states).delete(eq(NON_RANDOM_UUID)); verifyNoMoreInteractions(states); verifyZeroInteractions(httpCommunication); } @Test(expected = YggdrasilException.class) public void testBadFileURL() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(true); RequestHandlerContext context = new RequestHandlerContext(bitrepository, config, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); PreservationRequest requestWithBadURI = makeRequest(); requestWithBadURI.File_UUID = "Random File UUID"; requestWithBadURI.Content_URI = "http://127.0.0.1/" + (new Date()).getTime(); prh.handleRequest(requestWithBadURI); } @Test public void testMissingCollection() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); // Do not return an array with the default collection in it. when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList("")); RequestHandlerContext context = new RequestHandlerContext(bitrepository, config, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); prh.handleRequest(request); verify(updater).sendPreservationResponseWithSpecificDetails(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_REQUEST_FAILED), anyString()); verifyNoMoreInteractions(updater); verify(bitrepository).getKnownCollections(); verifyNoMoreInteractions(states, bitrepository); verifyZeroInteractions(httpCommunication); } @Test public void testFailedUpload() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(false); RequestHandlerContext context = new RequestHandlerContext(bitrepository, config, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); prh.handleRequest(request); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_REQUEST_RECEIVED)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_METADATA_PACKAGED_SUCCESSFULLY)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_COMPLETE)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_WAITING_FOR_MORE_DATA)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_UPLOAD_FAILURE)); verifyNoMoreInteractions(updater); verify(states, times(3)).putPreservationRecord(eq(NON_RANDOM_UUID), any(PreservationRequestState.class)); verify(states).delete(eq(NON_RANDOM_UUID)); verifyNoMoreInteractions(states); verify(bitrepository).getKnownCollections(); verify(bitrepository).uploadFile(any(File.class), eq(DEFAULT_COLLECTION)); verifyZeroInteractions(httpCommunication); } @Test public void testTimeout() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); // Setup for huge warc-size, low wait-limit and condition checking interval. YggdrasilConfig spyConfig = spy(config); stub(spyConfig.getWarcSizeLimit()).toReturn(100000000L); stub(spyConfig.getUploadWaitLimit()).toReturn(100L); stub(spyConfig.getCheckWarcConditionInterval()).toReturn(100L); when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(true); RequestHandlerContext context = new RequestHandlerContext(bitrepository, spyConfig, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); prh.handleRequest(request); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_REQUEST_RECEIVED)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_METADATA_PACKAGED_SUCCESSFULLY)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_COMPLETE)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_WAITING_FOR_MORE_DATA)); verify(updater, timeout(1500)).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_UPLOAD_SUCCESS)); verifyNoMoreInteractions(updater); verify(states, timeout(500)).delete(eq(NON_RANDOM_UUID)); verify(states, times(3)).putPreservationRecord(eq(NON_RANDOM_UUID), any(PreservationRequestState.class)); verifyNoMoreInteractions(states); verify(bitrepository).getKnownCollections(); verify(bitrepository).uploadFile(any(File.class), eq(DEFAULT_COLLECTION)); verifyNoMoreInteractions(bitrepository); verifyZeroInteractions(httpCommunication); } @Test public void testMultipleWarcFiles() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); String uuid1 = "UUID-1"; String uuid2 = "UUID-2"; PreservationRequest message = new PreservationRequest(); message.metadata = request.metadata; message.Model = request.Model; message.Preservation_profile = request.Preservation_profile; message.Valhal_ID = request.Valhal_ID; message.UUID = uuid1; when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(true); RequestHandlerContext context = new RequestHandlerContext(bitrepository, config, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); prh.handleRequest(message); verify(states, times(3)).putPreservationRecord(eq(uuid1), any(PreservationRequestState.class)); verify(states).delete(eq(uuid1)); message.UUID = uuid2; prh.handleRequest(message); verify(states, times(3)).putPreservationRecord(eq(uuid2), any(PreservationRequestState.class)); verify(states).delete(eq(uuid2)); verify(bitrepository, times(2)).uploadFile(any(File.class), eq(DEFAULT_COLLECTION)); verifyZeroInteractions(httpCommunication); } @Test public void testMultipleRecords() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); // Setup for huge warc-size, low wait-limit and condition checking interval. YggdrasilConfig spyConfig = spy(config); stub(spyConfig.getWarcSizeLimit()).toReturn(100000000L); stub(spyConfig.getUploadWaitLimit()).toReturn(100L); stub(spyConfig.getCheckWarcConditionInterval()).toReturn(100L); when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(true); RequestHandlerContext context = new RequestHandlerContext(bitrepository, spyConfig, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); prh.handleRequest(request); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_REQUEST_RECEIVED)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_METADATA_PACKAGED_SUCCESSFULLY)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_COMPLETE)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_WAITING_FOR_MORE_DATA)); verify(updater, timeout(1500)).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_UPLOAD_SUCCESS)); verifyNoMoreInteractions(updater); verify(states, timeout(1500)).delete(eq(NON_RANDOM_UUID)); verify(states, times(3)).putPreservationRecord(eq(NON_RANDOM_UUID), any(PreservationRequestState.class)); verifyNoMoreInteractions(states); verify(bitrepository).getKnownCollections(); verify(bitrepository).uploadFile(any(File.class), eq(DEFAULT_COLLECTION)); verifyNoMoreInteractions(bitrepository); verifyZeroInteractions(httpCommunication); } @Test public void testContentFileRequest() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); PreservationRequest request = makeRequest(); request.Model = "contentfile"; request.File_UUID = NON_RANDOM_FILE_UUID; request.Content_URI = "http://localhost/test.txt"; request.metadata = MetadataContentUtils.getExampleContentFileMetadata(); String payloadText = "Content file content"; HttpPayload payload = new HttpPayload(new ByteArrayInputStream(payloadText.getBytes()), null, "application/octetstream", (long) payloadText.length(), testFileDir); when(httpCommunication.get(anyString())).thenReturn(payload); when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(true); RequestHandlerContext context = new RequestHandlerContext(bitrepository, config, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); prh.handleRequest(request); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_REQUEST_RECEIVED)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_RESOURCES_DOWNLOAD_SUCCESS)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_METADATA_PACKAGED_SUCCESSFULLY)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_RESOURCES_PACKAGE_SUCCESS)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_COMPLETE)); verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_WAITING_FOR_MORE_DATA)); verify(updater, timeout(1500)).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_PACKAGE_UPLOAD_SUCCESS)); verifyNoMoreInteractions(updater); verify(states, times(4)).putPreservationRecord(eq(NON_RANDOM_UUID), any(PreservationRequestState.class)); verify(states, timeout(1500)).delete(eq(NON_RANDOM_UUID)); verifyNoMoreInteractions(states); verify(bitrepository).getKnownCollections(); verify(bitrepository).uploadFile(any(File.class), eq(DEFAULT_COLLECTION)); verifyNoMoreInteractions(bitrepository); verify(httpCommunication).get(anyString()); verifyNoMoreInteractions(httpCommunication); } @Test public void testContentFileDownloadFailure() throws Exception { StateDatabase states = mock(StateDatabase.class); Bitrepository bitrepository = mock(Bitrepository.class); RemotePreservationStateUpdater updater = getMockUpdater(); HttpCommunication httpCommunication = mock(HttpCommunication.class); PreservationRequest request = makeRequest(); request.Model = "contentfile"; request.File_UUID = NON_RANDOM_FILE_UUID; request.Content_URI = "http://localhost/test.txt"; request.metadata = MetadataContentUtils.getExampleContentFileMetadata(); when(httpCommunication.get(anyString())).thenReturn(null); when(bitrepository.getKnownCollections()).thenReturn(Arrays.asList(DEFAULT_COLLECTION)); when(bitrepository.uploadFile(any(File.class), anyString())).thenReturn(true); RequestHandlerContext context = new RequestHandlerContext(bitrepository, config, states, updater, httpCommunication); PreservationRequestHandler prh = new PreservationRequestHandler(context, models); try { prh.handleRequest(request); Assert.fail(); } catch (YggdrasilException e) { // Expected } verify(updater).sendPreservationResponse(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_REQUEST_RECEIVED)); verify(updater).sendPreservationResponseWithSpecificDetails(any(PreservationRequestState.class), eq(PreservationState.PRESERVATION_RESOURCES_DOWNLOAD_FAILURE), anyString()); verifyNoMoreInteractions(updater); verify(states).putPreservationRecord(eq(NON_RANDOM_UUID), any(PreservationRequestState.class)); verify(states, timeout(1500)).delete(eq(NON_RANDOM_UUID)); verifyNoMoreInteractions(states); verify(bitrepository).getKnownCollections(); verifyNoMoreInteractions(bitrepository); verify(httpCommunication).get(anyString()); verifyNoMoreInteractions(httpCommunication); } public static PreservationRequest makeRequest() { PreservationRequest request = new PreservationRequest(); request.Content_URI = null; request.File_UUID = null; request.metadata = MetadataContentUtils.getExampleInstanceMetadata(); request.Model = "instance"; request.Preservation_profile = DEFAULT_COLLECTION; request.UUID = NON_RANDOM_UUID; request.Valhal_ID = "ID"; return request; } public static RemotePreservationStateUpdater getMockUpdater() throws YggdrasilException { RemotePreservationStateUpdater res = mock(RemotePreservationStateUpdater.class); // Needed for setting of the preservation import state by the updater, when failing. doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { PreservationRequestState prs = (PreservationRequestState) invocation.getArguments()[0]; PreservationState newState = (PreservationState) invocation.getArguments()[1]; prs.setState(newState); return null; } }).when(res).sendPreservationResponse(any(PreservationRequestState.class), any(PreservationState.class)); return res; } }