/** * Copyright 2008 The University of North Carolina at Chapel Hill * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package edu.unc.lib.dl.services; import static edu.unc.lib.dl.util.ContentModelHelper.Datastream.MD_CONTENTS; import static edu.unc.lib.dl.util.ContentModelHelper.Datastream.RELS_EXT; import static edu.unc.lib.dl.util.ContentModelHelper.Model.CONTAINER; import static edu.unc.lib.dl.util.ContentModelHelper.Relationship.contains; import static edu.unc.lib.dl.util.ContentModelHelper.Relationship.removedChild; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; import java.io.InputStream; import java.util.Arrays; import java.util.List; import java.util.Set; import org.apache.commons.io.IOUtils; import org.jdom2.Document; import org.jdom2.Element; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatcher; import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import edu.unc.lib.dl.acl.util.AccessGroupSet; import edu.unc.lib.dl.acl.util.Permission; import edu.unc.lib.dl.fedora.AccessClient; import edu.unc.lib.dl.fedora.ClientUtils; import edu.unc.lib.dl.fedora.DatastreamDocument; import edu.unc.lib.dl.fedora.FedoraAccessControlService; import edu.unc.lib.dl.fedora.FedoraException; import edu.unc.lib.dl.fedora.ManagementClient; import edu.unc.lib.dl.fedora.OptimisticLockException; import edu.unc.lib.dl.fedora.PID; import edu.unc.lib.dl.fedora.types.Datastream; import edu.unc.lib.dl.fedora.types.MIMETypedStream.Header; import edu.unc.lib.dl.fedora.types.Property; import edu.unc.lib.dl.ingest.IngestException; import edu.unc.lib.dl.util.TripleStoreQueryService; import edu.unc.lib.dl.xml.JDOMNamespaceUtil; import edu.unc.lib.dl.xml.JDOMQueryUtil; /** * @author bbpennel * @date Jan 30, 2015 */ public class DigitalObjectManagerMoveTest { private DigitalObjectManagerImpl digitalMan; @Mock private ManagementClient managementClient; @Mock private AccessClient accessClient; @Mock private TripleStoreQueryService tripleStoreQueryService; @Mock private FedoraAccessControlService aclService; private Header dsHeaders; private PID destPID; private final PID source1PID = new PID("uuid:source1"); private final PID source2PID = new PID("uuid:source2"); @Before public void setUp() throws Exception { initMocks(this); dsHeaders = mock(Header.class); when(dsHeaders.getProperty()).thenReturn(Arrays.asList(new Property[0])); digitalMan = new DigitalObjectManagerImpl(); digitalMan.setAvailable(true, "available"); digitalMan.setAccessClient(accessClient); digitalMan.setForwardedManagementClient(null); digitalMan.setManagementClient(managementClient); digitalMan.setAclService(aclService); digitalMan.setTripleStoreQueryService(tripleStoreQueryService); destPID = new PID("uuid:destination"); when(tripleStoreQueryService.lookupRepositoryAncestorPids(destPID)).thenReturn( Arrays.asList(new PID("uuid:Collections"), new PID("uuid:collection"))); when(tripleStoreQueryService.lookupContentModels(destPID)).thenReturn(Arrays.asList(CONTAINER.getURI())); makeMatcherPair("/fedora/containerRELSEXT2.xml", destPID); when(aclService.hasAccess(any(PID.class), any(AccessGroupSet.class), eq(Permission.addRemoveContents))) .thenReturn(true); } @Test(expected = IngestException.class) public void moveParentIntoChildTest() throws Exception { List<PID> moving = Arrays.asList(new PID("uuid:child1"), new PID("uuid:collection")); digitalMan.move(moving, destPID, "user", ""); } @Test(expected = IngestException.class) public void destinationDoesNotExistTest() throws Exception { when(tripleStoreQueryService.lookupRepositoryAncestorPids(destPID)).thenReturn(null); List<PID> moving = Arrays.asList(new PID("uuid:child1"), new PID("uuid:child2")); digitalMan.move(moving, destPID, "user", ""); } public static class PairedMatcher extends ArgumentMatcher<Document> { public Document lastMatchedDoc; @Override public boolean matches(Object argument) { lastMatchedDoc = (Document) argument; return true; } } private class PairedAnswer implements Answer<Document> { public PairedAnswer(Document startingValue, PairedMatcher matcher) { this.startingValue = startingValue; this.matcher = matcher; } private final Document startingValue; private final PairedMatcher matcher; @Override public Document answer(InvocationOnMock invocation) throws Throwable { if (matcher.lastMatchedDoc == null) return startingValue; return matcher.lastMatchedDoc.clone(); } } private DatastreamDocument makeMatcherPair(String relsPath, PID targetPID) throws Exception { InputStream relsExtStream = this.getClass().getResourceAsStream(relsPath); DatastreamDocument dsDoc = mock(DatastreamDocument.class); Document doc = ClientUtils.parseXML(IOUtils.toByteArray(relsExtStream)); // PairedMatcher allows future calls to get the document to contain the last modified version PairedMatcher sourceRelsExtMatcher = new PairedMatcher(); PairedAnswer sourceRelsExtAnswer = new PairedAnswer(doc, sourceRelsExtMatcher); when(dsDoc.getDocument()).thenAnswer(sourceRelsExtAnswer); when(managementClient.getRELSEXTWithRetries(eq(targetPID))) .thenReturn(dsDoc); doNothing().when(managementClient) .modifyDatastream(eq(targetPID), eq(RELS_EXT.getName()), anyString(), anyString(), argThat(sourceRelsExtMatcher)); return dsDoc; } @Test public void oneSourceTest() throws Exception { makeMatcherPair("/fedora/containerRELSEXT1.xml", source1PID); List<PID> moving = Arrays.asList(new PID("uuid:child1"), new PID("uuid:child5")); when(tripleStoreQueryService.queryResourceIndex(anyString())) .thenReturn(Arrays.asList(Arrays.asList(source1PID.getPid()))); digitalMan.move(moving, destPID, "user", ""); verify(managementClient, times(2)).getRELSEXTWithRetries(eq(source1PID)); verifySourceMoved(source1PID, moving, 10); verifyDestinationMoved(moving, 9); } private void makeDatastream(String contentPath, String datastream, PID pid) throws Exception { InputStream mdContentsStream = this.getClass().getResourceAsStream(contentPath); DatastreamDocument dsDoc = mock(DatastreamDocument.class); Document doc = ClientUtils.parseXML(IOUtils.toByteArray(mdContentsStream)); when(dsDoc.getDocument()).thenReturn(doc); when(managementClient.getXMLDatastreamIfExists(eq(pid), eq(datastream))) .thenReturn(dsDoc); } @Test public void oneSourceWithMDContents() throws Exception { makeDatastream("/fedora/mdContents1.xml", MD_CONTENTS.getName(), source1PID); oneSourceTest(); verify(managementClient).modifyDatastream(eq(source1PID), eq(MD_CONTENTS.getName()), anyString(), anyString(), any(Document.class)); } @Test public void oneSourceLockFailTest() throws Exception { makeDatastream("/fedora/mdContents1.xml", MD_CONTENTS.getName(), source1PID); PairedMatcher sourceRelsExtMatcher = new PairedMatcher(); InputStream relsExtStream = this.getClass().getResourceAsStream("/fedora/containerRELSEXT1.xml"); Document doc = ClientUtils.parseXML(IOUtils.toByteArray(relsExtStream)); PairedAnswer sourceRelsExtAnswer = new PairedAnswer(doc, sourceRelsExtMatcher); // Throw locking exception on first attempt to rewrite source RELS-EXT doThrow(new OptimisticLockException("")) .doNothing() .when(managementClient) .modifyDatastream(eq(source1PID), eq(RELS_EXT.getName()), anyString(), anyString(), argThat(sourceRelsExtMatcher)); doThrow(new OptimisticLockException("")) .doNothing() .when(managementClient) .modifyDatastream(eq(destPID), eq(RELS_EXT.getName()), anyString(), anyString(), any(Document.class)); doThrow(new OptimisticLockException("")) .doNothing() .when(managementClient) .modifyDatastream(eq(source1PID), eq(MD_CONTENTS.getName()), anyString(), anyString(), any(Document.class)); DatastreamDocument dsDoc = mock(DatastreamDocument.class); when(dsDoc.getDocument()).thenAnswer(sourceRelsExtAnswer); when(managementClient.getRELSEXTWithRetries(eq(source1PID))) .thenReturn(dsDoc); Datastream ds = mock(Datastream.class); when(ds.getCreateDate()).thenReturn(new String()); when(managementClient.getDatastream(eq(source1PID), eq(RELS_EXT.getName()))).thenReturn(ds); List<PID> moving = Arrays.asList(new PID("uuid:child1"), new PID("uuid:child5")); when(tripleStoreQueryService.queryResourceIndex(anyString())) .thenReturn(Arrays.asList(Arrays.asList(source1PID.getPid()))); digitalMan.move(moving, destPID, "user", ""); verify(managementClient, times(3)).getRELSEXTWithRetries(eq(source1PID)); ArgumentCaptor<Document> sourceRelsExtUpdateCaptor = ArgumentCaptor.forClass(Document.class); verify(managementClient, times(3)).modifyDatastream(eq(source1PID), eq(RELS_EXT.getName()), anyString(), anyString(), sourceRelsExtUpdateCaptor.capture()); List<Document> sourceRelsAnswers = sourceRelsExtUpdateCaptor.getAllValues(); // Verify that the initial source RELS-EXT update is repeated after failure Document sourceRelsExt = sourceRelsAnswers.get(0); Set<PID> removed = JDOMQueryUtil.getRelationSet(sourceRelsExt.getRootElement(), removedChild); assertEquals("Child tombstones should still be present", 2, removed.size()); sourceRelsExt = sourceRelsAnswers.get(1); removed = JDOMQueryUtil.getRelationSet(sourceRelsExt.getRootElement(), removedChild); assertEquals("Child tombstones should still be present on second try", 2, removed.size()); // Check that tombstones were cleaned up by the end of the operation Document cleanRelsExt = sourceRelsAnswers.get(2); Set<PID> children = JDOMQueryUtil.getRelationSet(cleanRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in source container after cleanup", 10, children.size()); removed = JDOMQueryUtil.getRelationSet(cleanRelsExt.getRootElement(), removedChild); assertEquals("Child tombstones not cleaned up", 0, removed.size()); // Verify that the destination had the moved children added to it ArgumentCaptor<Document> destRelsExtUpdateCaptor = ArgumentCaptor.forClass(Document.class); verify(managementClient, times(2)).modifyDatastream(eq(destPID), eq(RELS_EXT.getName()), anyString(), anyString(), destRelsExtUpdateCaptor.capture()); Document destRelsExt = destRelsExtUpdateCaptor.getValue(); children = JDOMQueryUtil.getRelationSet(destRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in destination container after moved", 9, children.size()); assertTrue("Moved children were not present in destination", children.containsAll(moving)); verify(managementClient, times(2)).modifyDatastream(eq(source1PID), eq(MD_CONTENTS.getName()), anyString(), anyString(), any(Document.class)); } @Test public void multiSourceTest() throws Exception { List<PID> moving = Arrays.asList(new PID("uuid:child1"), new PID("uuid:child32")); makeMatcherPair("/fedora/containerRELSEXT1.xml", source1PID); makeMatcherPair("/fedora/containerRELSEXT3.xml", source2PID); when(tripleStoreQueryService.queryResourceIndex(anyString())) .thenReturn(Arrays.asList(Arrays.asList(source1PID.getPid()))) .thenReturn(Arrays.asList(Arrays.asList(source2PID.getPid()))); digitalMan.move(moving, destPID, "user", ""); verify(managementClient, times(2)).getRELSEXTWithRetries(eq(source1PID)); verify(managementClient, times(2)).getRELSEXTWithRetries(eq(source2PID)); verifySourceMoved(source1PID, Arrays.asList(new PID("uuid:child1")), 11); verifySourceMoved(source2PID, Arrays.asList(new PID("uuid:child32")), 1); verifyDestinationMoved(moving, 9); } @Test public void rollbackNoProblemsTest() throws Exception { // Create a document for the source which has no removedChildren DatastreamDocument dsDoc = mock(DatastreamDocument.class); Document doc = mock(Document.class); when(dsDoc.getDocument()).thenReturn(doc); when(doc.getRootElement()).thenReturn( new Element("RDF", JDOMNamespaceUtil.RDF_NS) .addContent(new Element("Description", JDOMNamespaceUtil.RDF_NS))); when(managementClient.getRELSEXTWithRetries(any(PID.class))) .thenReturn(dsDoc); List<PID> moving = Arrays.asList(new PID("uuid:child1"), new PID("uuid:child5")); digitalMan.rollbackMove(source1PID, moving); // Verify that it doesn't try to change anything when there are no leftover tombstones verify(managementClient, never()).modifyDatastream(eq(destPID), eq(RELS_EXT.getName()), anyString(), anyString(), any(Document.class)); } @Test public void rollbackTest() throws Exception { makeMatcherPair("/fedora/containerRELSEXT1.xml", source1PID); List<PID> moving = Arrays.asList(new PID("uuid:child1"), new PID("uuid:child5")); when(tripleStoreQueryService.fetchContainer(any(PID.class))).thenReturn(source1PID).thenReturn(source1PID) .thenReturn(null).thenReturn(null); when(tripleStoreQueryService.queryResourceIndex(anyString())) .thenReturn(Arrays.asList(Arrays.asList(source1PID.getPid()))) .thenReturn(Arrays.asList(Arrays.asList(source1PID.getPid()))) .thenReturn(null).thenReturn(null); when(managementClient.getRELSEXTWithRetries(eq(destPID))).thenReturn(null); try { digitalMan.move(moving, destPID, "user", ""); fail(); } catch (IngestException e) { // Expected } ArgumentCaptor<Document> sourceRelsExtUpdateCaptor = ArgumentCaptor.forClass(Document.class); // There should have been three updates to the source RELS-EXT, the initial, rollback the moved, and cleanup verify(managementClient, times(3)).modifyDatastream(eq(source1PID), eq(RELS_EXT.getName()), anyString(), anyString(), sourceRelsExtUpdateCaptor.capture()); List<Document> sourceRelsAnswers = sourceRelsExtUpdateCaptor.getAllValues(); // Check the state of the source after removal but before cleanup Document sourceRelsExt = sourceRelsAnswers.get(0); Set<PID> children = JDOMQueryUtil.getRelationSet(sourceRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in source container after move", 10, children.size()); Set<PID> removed = JDOMQueryUtil.getRelationSet(sourceRelsExt.getRootElement(), removedChild); assertEquals("Moved child gravestones not correctly set in source container", 2, removed.size()); // Children should all be back as contains statements as part of the rollback Document rbRelsExt = sourceRelsAnswers.get(1); children = JDOMQueryUtil.getRelationSet(rbRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in source container after first rollback", 12, children.size()); removed = JDOMQueryUtil.getRelationSet(rbRelsExt.getRootElement(), removedChild); assertEquals("Child tombstones should still be present", 2, removed.size()); // Should be back to the original set of children relations by the end of rolling back Document cleanRelsExt = sourceRelsAnswers.get(2); children = JDOMQueryUtil.getRelationSet(cleanRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in source container after rollback cleanup", 12, children.size()); removed = JDOMQueryUtil.getRelationSet(cleanRelsExt.getRootElement(), removedChild); assertEquals("Child tombstones not cleaned up", 0, removed.size()); } @Test public void rollbackAfterDestinationUpdateTest() throws Exception { DatastreamDocument dsDoc = makeMatcherPair("/fedora/containerRELSEXT1.xml", source1PID); makeMatcherPair("/fedora/containerRELSEXT2.xml", destPID); List<PID> moving = Arrays.asList(new PID("uuid:child1"), new PID("uuid:child5")); when(tripleStoreQueryService.queryResourceIndex(anyString())) .thenReturn(Arrays.asList(Arrays.asList(source1PID.getPid()))) .thenReturn(Arrays.asList(Arrays.asList(source1PID.getPid()))) .thenReturn(Arrays.asList(Arrays.asList(destPID.getPid()))) .thenReturn(Arrays.asList(Arrays.asList(destPID.getPid()))); // Second attempt to get the source RELS-EXT will return null to trigger rollback when (managementClient.getRELSEXTWithRetries(eq(source1PID))) .thenReturn(dsDoc).thenThrow(new FedoraException("")).thenReturn(dsDoc); try { digitalMan.move(moving, destPID, "user", ""); fail(); } catch (IngestException e) { // Expected } ArgumentCaptor<Document> sourceRelsExtUpdateCaptor = ArgumentCaptor.forClass(Document.class); // There should have been three updates to the source RELS-EXT, the initial, rollback the moved, and cleanup verify(managementClient, times(3)).modifyDatastream(eq(source1PID), eq(RELS_EXT.getName()), anyString(), anyString(), sourceRelsExtUpdateCaptor.capture()); List<Document> sourceRelsAnswers = sourceRelsExtUpdateCaptor.getAllValues(); // Check the state of the source after removal but before cleanup Document sourceRelsExt = sourceRelsAnswers.get(0); Set<PID> children = JDOMQueryUtil.getRelationSet(sourceRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in source container after move", 10, children.size()); Set<PID> removed = JDOMQueryUtil.getRelationSet(sourceRelsExt.getRootElement(), removedChild); assertEquals("Moved child gravestones not correctly set in source container", 2, removed.size()); // Should be back to the original set of children relations by the end of rolling back Document cleanRelsExt = sourceRelsAnswers.get(2); children = JDOMQueryUtil.getRelationSet(cleanRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in source container after rollback cleanup", 12, children.size()); removed = JDOMQueryUtil.getRelationSet(cleanRelsExt.getRootElement(), removedChild); assertEquals("Child tombstones not cleaned up", 0, removed.size()); ArgumentCaptor<Document> destRelsExtUpdateCaptor = ArgumentCaptor.forClass(Document.class); verify(managementClient, times(2)).modifyDatastream(eq(destPID), eq(RELS_EXT.getName()), anyString(), anyString(), destRelsExtUpdateCaptor.capture()); List<Document> destRelsAnswers = destRelsExtUpdateCaptor.getAllValues(); // Check the state of the source after removal but before cleanup Document destRelsExt = destRelsAnswers.get(0); children = JDOMQueryUtil.getRelationSet(destRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in destination container after move", 9, children.size()); // Should be back to the original set of children relations by the end of rolling back Document destRBRelsExt = destRelsAnswers.get(1); children = JDOMQueryUtil.getRelationSet(destRBRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in destination container after rollback", 7, children.size()); } @Test public void multipleContainersTest() throws Exception { makeMatcherPair("/fedora/containerRELSEXT1.xml", source1PID); makeMatcherPair("/fedora/containerRELSEXT1.xml", source2PID); List<PID> moving = Arrays.asList(new PID("uuid:child1")); when(tripleStoreQueryService.queryResourceIndex(anyString())) .thenReturn(Arrays.asList(Arrays.asList(source1PID.getPid()), Arrays.asList(source2PID.getPid()))); digitalMan.move(moving, destPID, "user", ""); verifySourceMoved(source1PID, moving, 11); verifySourceMoved(source2PID, moving, 11); verifyDestinationMoved(moving, 8); } private void verifySourceMoved(PID sourcePID, List<PID> moving, int sourceContainsCount) throws Exception { ArgumentCaptor<Document> sourceRelsExtUpdateCaptor = ArgumentCaptor.forClass(Document.class); verify(managementClient, times(2)).modifyDatastream(eq(sourcePID), eq(RELS_EXT.getName()), anyString(), anyString(), sourceRelsExtUpdateCaptor.capture()); List<Document> sourceRelsAnswers = sourceRelsExtUpdateCaptor.getAllValues(); // Check the state of the source after removal but before cleanup Document sourceRelsExt = sourceRelsAnswers.get(0); Set<PID> children = JDOMQueryUtil.getRelationSet(sourceRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in source container after move", sourceContainsCount, children.size()); Set<PID> removed = JDOMQueryUtil.getRelationSet(sourceRelsExt.getRootElement(), removedChild); assertEquals("Moved child gravestones not correctly set in source container", moving.size(), removed.size()); // Check that tombstones were cleaned up by the end of the operation Document cleanRelsExt = sourceRelsAnswers.get(1); children = JDOMQueryUtil.getRelationSet(cleanRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in source container after cleanup", sourceContainsCount, children.size()); removed = JDOMQueryUtil.getRelationSet(cleanRelsExt.getRootElement(), removedChild); assertEquals("Child tombstones not cleaned up", 0, removed.size()); assertFalse("Moved children were still present in source", children.containsAll(moving)); } private void verifyDestinationMoved(List<PID> moving, int destinationCount) throws Exception { // Verify that the destination had the moved children added to it verify(managementClient).getRELSEXTWithRetries(eq(destPID)); ArgumentCaptor<Document> destRelsExtUpdateCaptor = ArgumentCaptor.forClass(Document.class); verify(managementClient).modifyDatastream(eq(destPID), eq(RELS_EXT.getName()), anyString(), anyString(), destRelsExtUpdateCaptor.capture()); Document destRelsExt = destRelsExtUpdateCaptor.getValue(); Set<PID> children = JDOMQueryUtil.getRelationSet(destRelsExt.getRootElement(), contains); assertEquals("Incorrect number of children in destination container after moved", destinationCount, children.size()); assertTrue("Moved children were not present in destination", children.containsAll(moving)); } }