/** * Copyright 2008 The University of North Carolina at Chapel Hill * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package edu.unc.lib.dl.services; import static edu.unc.lib.dl.util.ContentModelHelper.Datastream.MD_CONTENTS; import static edu.unc.lib.dl.util.ContentModelHelper.Datastream.RELS_EXT; import static edu.unc.lib.dl.util.ContentModelHelper.Relationship.contains; import static edu.unc.lib.dl.util.ContentModelHelper.Relationship.removedChild; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import javax.xml.transform.Source; import javax.xml.transform.TransformerException; import javax.xml.transform.stream.StreamSource; import org.jdom2.Document; import org.jdom2.Element; import org.jdom2.JDOMException; import org.jdom2.Namespace; import org.jdom2.input.SAXBuilder; import org.jdom2.output.XMLOutputter; import org.jdom2.transform.JDOMSource; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import edu.unc.lib.dl.acl.util.AccessGroupSet; import edu.unc.lib.dl.acl.util.GroupsThreadStore; import edu.unc.lib.dl.acl.util.Permission; import edu.unc.lib.dl.fedora.AccessClient; import edu.unc.lib.dl.fedora.AuthorizationException; import edu.unc.lib.dl.fedora.DatastreamDocument; import edu.unc.lib.dl.fedora.FedoraAccessControlService; import edu.unc.lib.dl.fedora.FedoraException; import edu.unc.lib.dl.fedora.ManagementClient; import edu.unc.lib.dl.fedora.ManagementClient.ChecksumType; import edu.unc.lib.dl.fedora.ManagementClient.Format; import edu.unc.lib.dl.fedora.ManagementClient.State; import edu.unc.lib.dl.fedora.NotFoundException; import edu.unc.lib.dl.fedora.OptimisticLockException; import edu.unc.lib.dl.fedora.PID; import edu.unc.lib.dl.fedora.types.MIMETypedStream; import edu.unc.lib.dl.ingest.IngestException; import edu.unc.lib.dl.schematron.SchematronValidator; import edu.unc.lib.dl.update.UpdateException; import edu.unc.lib.dl.util.Checksum; import edu.unc.lib.dl.util.ContainerContentsHelper; import edu.unc.lib.dl.util.ContentModelHelper; import edu.unc.lib.dl.util.ContentModelHelper.CDRProperty; import edu.unc.lib.dl.util.ContentModelHelper.Datastream; import edu.unc.lib.dl.util.ContentModelHelper.Model; import edu.unc.lib.dl.util.ContentModelHelper.Relationship; import edu.unc.lib.dl.util.IllegalRepositoryStateException; import edu.unc.lib.dl.util.IndexingActionType; import edu.unc.lib.dl.util.PremisEventLogger; import edu.unc.lib.dl.util.PremisEventLogger.Type; import edu.unc.lib.dl.util.ResourceType; import edu.unc.lib.dl.util.TripleStoreQueryService; import edu.unc.lib.dl.xml.FOXMLJDOMUtil; import edu.unc.lib.dl.xml.FOXMLJDOMUtil.ObjectProperty; import edu.unc.lib.dl.xml.JDOMNamespaceUtil; import edu.unc.lib.dl.xml.JDOMQueryUtil; import edu.unc.lib.dl.xml.ModsXmlHelper; /** * This class orchestrates the transactions that modify repository objects and update ancillary services. * * @author count0 * */ public class DigitalObjectManagerImpl implements DigitalObjectManager { private static final Logger log = LoggerFactory.getLogger(DigitalObjectManagerImpl.class); private boolean available = false; private String availableMessage = "The repository manager is not available yet."; private AccessClient accessClient = null; private ManagementClient forwardedManagementClient = null; private ManagementClient managementClient = null; private FedoraAccessControlService aclService = null; private OperationsMessageSender operationsMessageSender = null; private TripleStoreQueryService tripleStoreQueryService = null; private SchematronValidator schematronValidator = null; private PID collectionsPid = null; public synchronized void setAvailable(boolean available, String message) { this.available = available; this.availableMessage = message; } public synchronized void setAvailable(boolean available) { this.setAvailable(available, "Repository undergoing maintenance, please contact staff for more information."); } public SchematronValidator getSchematronValidator() { return schematronValidator; } public void setSchematronValidator(SchematronValidator schematronValidator) { this.schematronValidator = schematronValidator; } public DigitalObjectManagerImpl() { } private void availableCheck() throws IngestException { if (!this.available) { throw new IngestException(this.availableMessage + " \nContact repository staff for assistance."); } } /** * @param lastKnownGoodTime * @param pids */ private void dumpRollbackInfo(DateTime lastKnownGoodTime, List<PID> pids, String reason) { StringBuffer sb = new StringBuffer(); sb.append("DATA CORRUPTION LOG:\n").append("REASON:").append(reason).append("\n") .append("LAST KNOWN GOOD TIME: ").append(lastKnownGoodTime.toString()).append("\n").append(pids.size()) .append(" FEDORA PIDS CREATED OR MODIFIED:\n"); for (PID p : pids) { sb.append(p.getPid()).append("\n"); } log.error(sb.toString()); } @Override public void editResourceType(List<PID> subjects, ResourceType newType, String user) throws UpdateException { if (newType == null || ResourceType.File.equals(newType)) { throw new UpdateException("Cannot edit to type " + newType + ", operation not supported"); } // Check that the user has permissions to add/remove from all sources and the destination AccessGroupSet groups = GroupsThreadStore.getGroups(); for (PID subject : subjects) { if (!aclService.hasAccess(subject, groups, Permission.editResourceType)) { throw new UpdateException("Insufficient permissions to perform edit type", new AuthorizationException( "Cannot complete edit type operation, user " + user + " does not have permission to modify " + subject)); } } for (PID subject : subjects) { do { try { DatastreamDocument relsExtResp = managementClient.getXMLDatastreamIfExists(subject, RELS_EXT.getName()); if (relsExtResp == null) { throw new UpdateException("Unable to retrieve RELS-EXT for " + subject + ", cannot change models"); } Document relsExt = relsExtResp.getDocument(); Element descriptionEl = relsExt.getDocument().getRootElement() .getChild("Description", JDOMNamespaceUtil.RDF_NS); String hasModelPredicate = ContentModelHelper.FedoraProperty.hasModel.getFragment(); Namespace hasModelNS = ContentModelHelper.FedoraProperty.hasModel.getNamespace(); List<Element> matchingEls = descriptionEl.getChildren(hasModelPredicate, hasModelNS); // Determine what the starting content model profile is List<String> existingModels = new ArrayList<>(matchingEls.size()); for (Element modelEl : matchingEls) { existingModels.add(modelEl.getAttributeValue("resource", JDOMNamespaceUtil.RDF_NS)); } ResourceType existingType = ResourceType.getResourceTypeByContentModels(existingModels); // If the resource type hasn't changed from what is present in Fedora, then skip changing if (existingType.equals(newType)) { break; } // Validate that the conversion is allowed if (existingType.equals(ResourceType.File)) { // Can't convert from file currently throw new UpdateException("Cannot edit object " + subject + " from type File, operation not supported"); } // Remove existing content models for (ContentModelHelper.Model typeModels : existingType.getContentModels()) { Iterator<Element> matchingIt = matchingEls.iterator(); while (matchingIt.hasNext()) { Element match = matchingIt.next(); String existingValue = match.getAttributeValue("resource", JDOMNamespaceUtil.RDF_NS); if (existingValue != null && existingValue.equals(typeModels.toString())) { matchingIt.remove(); } } } // Add new content models for (ContentModelHelper.Model newModel : newType.getContentModels()) { Element newRelationEl = new Element(hasModelPredicate, hasModelNS); newRelationEl.setAttribute("resource", newModel.toString(), JDOMNamespaceUtil.RDF_NS); descriptionEl.addContent(newRelationEl); } if (log.isDebugEnabled()) { XMLOutputter outputter = new XMLOutputter(org.jdom2.output.Format.getPrettyFormat()); log.debug("Attempting to update RELS-EXT for {} to change models:\n{}", subject, outputter.outputString(relsExt)); } // Push the changes to the objects relations managementClient .modifyDatastream(subject, RELS_EXT.getName(), null, relsExtResp.getLastModified(), relsExt); // Add premis event for the resource type change PremisEventLogger logger = new PremisEventLogger(user); Element event = logger.logEvent(PremisEventLogger.Type.MIGRATION, "Changed resource type to " + newType.name(), subject); PremisEventLogger.addDetailedOutcome(event, "success", "Changed resource type from " + existingType.name() + " to " + newType.name(), null); this.forwardedManagementClient.writePremisEventsToFedoraObject(logger, subject); break; } catch (OptimisticLockException e) { log.debug("Unable to update RELS-EXT for {}, retrying", subject, e); } catch (FedoraException e) { throw new UpdateException("Error while updating relations for " + subject, e); } // Repeat rels-ext update if the source changed since the datastream was retrieved } while (true); } if (this.getOperationsMessageSender() != null) { this.getOperationsMessageSender().sendEditTypeOperation(user, subjects, newType); } } @Override public void editDefaultWebObject(List<PID> dwos, boolean clear, String user) throws UpdateException { Set<PID> modified = new HashSet<>(); for (PID dwo : dwos) { PID aggregate = this.tripleStoreQueryService.fetchParentByModel(dwo, Model.AGGREGATE_WORK); if (aggregate == null) { throw new UpdateException("Object " + dwo + " is not contained by an aggregate object"); } // Check that the user has sufficient permissions if (!aclService.hasAccess(aggregate, GroupsThreadStore.getGroups(), Permission.addRemoveContents)) { throw new UpdateException("Insufficient permissions to set default web object", new AuthorizationException( "Cannot set default web object, user " + user + " does not have permission to modify " + aggregate)); } do { try { log.debug("Assigning {} as the DWO for {}", dwo, aggregate); DatastreamDocument relsExtResp = managementClient.getXMLDatastreamIfExists(aggregate, RELS_EXT.getName()); if (relsExtResp == null) { throw new UpdateException("Unable to retrieve RELS-EXT for " + aggregate + ", cannot set default web object"); } Document relsExt = relsExtResp.getDocument(); Element descriptionEl = relsExt.getDocument().getRootElement() .getChild("Description", JDOMNamespaceUtil.RDF_NS); String predicate = CDRProperty.defaultWebObject.getPredicate(); Namespace ns = CDRProperty.defaultWebObject.getNamespace(); // Remove existing dwo relations and indicate their pids need updating. List<Element> dwoEls = descriptionEl.getChildren(predicate, ns); if (dwoEls != null) { Iterator<Element> dwoIt = dwoEls.iterator(); while (dwoIt.hasNext()) { Element dwoEl = dwoIt.next(); String existing = dwoEl.getAttributeValue("resource", JDOMNamespaceUtil.RDF_NS); modified.add(new PID(existing)); log.debug("Removing existing DWO of {} while assigning to {}", existing, aggregate); dwoIt.remove(); } } if (!clear) { // Add the new object Element newRelationEl = new Element(predicate, ns); newRelationEl.setAttribute("resource", dwo.getURI().toString(), JDOMNamespaceUtil.RDF_NS); descriptionEl.addContent(newRelationEl); log.debug("Added {} as DWO for {}", dwo, aggregate); } if (log.isDebugEnabled()) { XMLOutputter outputter = new XMLOutputter(org.jdom2.output.Format.getPrettyFormat()); log.debug("Attempting to update RELS-EXT for {} to set DWO:\n{}", aggregate, outputter.outputString(relsExt)); } // Push the changes to the objects relations managementClient .modifyDatastream(aggregate, RELS_EXT.getName(), null, relsExtResp.getLastModified(), relsExt); modified.add(aggregate); modified.add(dwo); break; } catch (OptimisticLockException e) { log.debug("Unable to update RELS-EXT for {}, retrying", aggregate, e); } catch (FedoraException e) { throw new UpdateException("Error while updating relations for " + aggregate, e); } // Repeat rels-ext update if the source changed since the datastream was retrieved } while (true); } // Send message that the action completed if (this.getOperationsMessageSender() != null) { this.getOperationsMessageSender().sendIndexingOperation(user, modified, IndexingActionType.SET_DEFAULT_WEB_OBJECT); } } /** * This method destroys a set of objects in Fedora, leaving no preservation data. It will update any ancillary * services and log delete events. * * @param pids * the PIDs of the objects to purge * @param message * the reason for the purge * @return a list of PIDs that were purged * @see edu.unc.lib.dl.services.DigitalObjectManager.purge() */ @Override public List<PID> delete(PID pid, String user, String message) throws IngestException, NotFoundException { availableCheck(); // Prevent deletion of the repository object and the collections object if (pid.equals(ContentModelHelper.Administrative_PID.REPOSITORY.getPID()) || pid.equals(collectionsPid)) throw new IllegalRepositoryStateException("Cannot delete administrative object: " + pid); List<PID> deleted = new ArrayList<PID>(); // FIXME disallow delete of "/admin" folder // TODO add protected delete method for force initializing // Get all children and store for deletion List<PID> toDelete = this.getTripleStoreQueryService().fetchAllContents(pid); toDelete.add(pid); // gathering delete set, checking for object relationships // Find all relationships which refer to the pid being deleted List<PID> refs = this.getReferencesToContents(pid); refs.removeAll(toDelete); if (refs.size() > 0) { StringBuffer s = new StringBuffer(); s.append("Cannot delete ").append(pid).append(" because it will break object references from these PIDs: "); for (PID b : refs) { s.append("\t").append(b); } throw new IngestException(s.toString()); } PID container = this.getTripleStoreQueryService().fetchContainer(pid); if (container == null) { throw new IllegalRepositoryStateException("Cannot find a container for the specified object: " + pid); } // begin transaction, must delete all content and modify parent or dump // rollback info PremisEventLogger logger = new PremisEventLogger(user); DateTime transactionStart = new DateTime(); Throwable thrown = null; List<PID> removed = new ArrayList<PID>(); removed.add(pid); try { // update container this.removeFromContainer(pid); Element event = logger.logEvent(PremisEventLogger.Type.DELETION, "Deleted " + deleted.size() + " contained object(s).", container); PremisEventLogger.addDetailedOutcome(event, "success", "Message: " + message, null); this.forwardedManagementClient.writePremisEventsToFedoraObject(logger, container); // delete object and all of its children for (PID obj : toDelete) { try { this.getManagementClient().purgeObject(obj, message, false); deleted.add(obj); } catch (NotFoundException e) { log.error("Delete set referenced an object that didn't exist: " + pid.getPid(), e); } } // Send message to message queue informing it of the deletion(s) if (this.getOperationsMessageSender() != null) { this.getOperationsMessageSender().sendRemoveOperation(user, container, removed, null); } } catch (FedoraException fault) { log.error("Fedora threw an unexpected fault while deleting " + pid.getPid(), fault); thrown = fault; } catch (RuntimeException e) { this.setAvailable(false); log.error("Fedora threw an unexpected runtime exception while deleting " + pid.getPid(), e); thrown = e; } finally { if (thrown != null && toDelete.size() > deleted.size()) { // some objects not deleted List<PID> missed = new ArrayList<PID>(); missed.addAll(toDelete); missed.removeAll(deleted); this.dumpRollbackInfo(transactionStart, missed, "Could not complete delete of " + pid.getPid() + ", please purge objects and check container " + container.getPid() + "."); } } if (thrown != null) { throw new IngestException("There was a problem completing the delete operation", thrown); } return deleted; } public AccessClient getAccessClient() { return accessClient; } ManagementClient getManagementClient() { return forwardedManagementClient; } /** * Generates a list of referring object PIDs. Dependent objects are currently defined as those objects that refer to * the specified pid in RELS-EXT other than it's container. * * @param pid * the object depended upon * @return a list of dependent object PIDs */ private List<PID> getReferencesToContents(PID pid) { List<PID> refs = this.getTripleStoreQueryService().fetchObjectReferences(pid); if (!ContentModelHelper.Administrative_PID.REPOSITORY.equals(pid)) { PID container = this.getTripleStoreQueryService().fetchContainer(pid); refs.remove(container); } return refs; } public TripleStoreQueryService getTripleStoreQueryService() { return tripleStoreQueryService; } /** * This must be called after properties are set. It checks for basic repository objects and throws a runtime * exception if they don't exist. */ public void init() { // throw a runtime exception? } /** * Just removes object from container, does not log this event. MUST finish operation or dump rollback info and * rethrow exception. * * @param pid * the PID of the object to remove * @return the PID of the old container * @throws FedoraException */ private PID removeFromContainer(PID pid) throws FedoraException { boolean relsextDone = false; PID parent = this.getTripleStoreQueryService().fetchContainer(pid); if (parent == null) { // Block removal of repo object if (ContentModelHelper.Administrative_PID.REPOSITORY.getPID().equals(pid)) return null; throw new NotFoundException("Found an object without a parent that is not the REPOSITORY"); } log.debug("removeFromContainer called on PID: " + parent.getPid()); try { // remove ir:contains statement to RELS-EXT relsextDone = this.getManagementClient().purgeObjectRelationship(parent, Relationship.contains.name(), Relationship.contains.getNamespace(), pid); if (relsextDone == false) { log.error("failed to purge relationship: " + parent + " contains " + pid); } // if the parent is a container, then make it orderly List<URI> cmtypes = this.getTripleStoreQueryService().lookupContentModels(parent); if (cmtypes.contains(ContentModelHelper.Model.CONTAINER.getURI())) { // edit Contents XML of parent container to append/insert try { Document newXML; Document oldXML; MIMETypedStream mts = this.getAccessClient().getDatastreamDissemination(parent, "MD_CONTENTS", null); try(ByteArrayInputStream bais = new ByteArrayInputStream(mts.getStream())) { oldXML = new SAXBuilder().build(bais); } newXML = ContainerContentsHelper.remove(oldXML, pid); this.getManagementClient().modifyInlineXMLDatastream(parent, "MD_CONTENTS", false, "removing child object from this container", new ArrayList<String>(), "List of Contents", newXML); } catch (NotFoundException e) { // MD_CONTENTS was not found, so we will assume this is an unordered container } } } catch (JDOMException e) { IllegalRepositoryStateException irs = new IllegalRepositoryStateException( "Invalid XML for container MD_CONTENTS: " + parent.getPid(), parent, e); log.error("Failed to parse XML", irs); throw irs; } catch (IOException e) { throw new Error("Should not get IOException for reading byte array input", e); } return parent; } public void setAccessClient(AccessClient accessClient) { this.accessClient = accessClient; } public void setForwardedManagementClient(ManagementClient managementClient) { this.forwardedManagementClient = managementClient; } public void setManagementClient(ManagementClient managementClient) { this.managementClient = managementClient; } public void setTripleStoreQueryService(TripleStoreQueryService tripleStoreQueryService) { this.tripleStoreQueryService = tripleStoreQueryService; } @Override public String updateSourceData(PID pid, String dsid, File newDataFile, String checksum, String label, String mimetype, String user, String message) throws IngestException { availableCheck(); String result = null; PremisEventLogger logger = new PremisEventLogger(user); // make sure the datastream is source data if (!this.getTripleStoreQueryService().isSourceData(pid, dsid)) { throw new IngestException("You can only update source datastreams. (marked as <pid-uri> <" + ContentModelHelper.CDRProperty.sourceData + "> <ds-uri> in RELS-EXT)"); } // compare checksum if one is supplied if (checksum != null) { try { String sum = new Checksum().getChecksum(newDataFile); if (!checksum.trim().toLowerCase().equals(sum.toLowerCase())) { throw new IngestException("MD5 calculated for file (" + sum + ") does not match supplied checksum (" + checksum + ")"); } else { logger.logEvent(Type.DIGITAL_SIGNATURE_VALIDATION, "Validated MD5 signature for updated source data file", pid, dsid); } } catch (FileNotFoundException e1) { throw new IngestException("New source data file not found", e1); } catch (IOException e1) { throw new IngestException("There was a problem read the new source data file", e1); } } String newref = null; try { // Upload the new file and then update the datastream newref = this.getManagementClient().upload(newDataFile); result = this.getManagementClient().modifyDatastreamByReference(pid, dsid, false, message, null, label, mimetype, checksum, ChecksumType.MD5, newref); // update PREMIS log logger.logEvent(PremisEventLogger.Type.INGESTION, message, pid, dsid); this.forwardedManagementClient.writePremisEventsToFedoraObject(logger, pid); } catch (FedoraException | IOException e) { throw new IngestException("Could not update the specified object.", e); } return result; } @Override public String updateDescription(PID pid, File newMODSFile, String checksum, String user, String message) throws IngestException { availableCheck(); String result = null; PremisEventLogger logger = new PremisEventLogger(user); // compare checksum if one is supplied if (checksum != null) { try { String sum = new Checksum().getChecksum(newMODSFile); if (!checksum.trim().toLowerCase().equals(sum.toLowerCase())) { throw new IngestException("MD5 calculated for file (" + sum + ") does not match supplied checksum (" + checksum + ")"); } else { logger.logEvent(Type.DIGITAL_SIGNATURE_VALIDATION, "Validated MD5 signature for updated descriptive metadata file", pid, "MD_DESCRIPTIVE"); } } catch (FileNotFoundException e1) { throw new IngestException("New MODS file not found", e1); } catch (IOException e1) { throw new IngestException("There was a problem reading the new MODS file", e1); } } // make sure the supplied XML is valid Element event = logger.logEvent(Type.VALIDATION, message, pid, "MD_DESCRIPTIVE"); Source source; try { source = new StreamSource(new FileInputStream(newMODSFile)); } catch (FileNotFoundException e1) { throw new Error("Unexpected exception", e1); } Document svrl = this.getSchematronValidator().validate(source, "vocabularies-mods"); if (!this.getSchematronValidator().hasFailedAssertions(svrl)) { PremisEventLogger.addDetailedOutcome(event, "MODS is valid", "The supplied MODS metadata meets all CDR vocabulary requirements.", null); } else { PremisEventLogger.addDetailedOutcome(event, "MODS is not valid", "The supplied MODS metadata does not meet CDR vocabulary requirements.", svrl.detachRootElement()); IngestException e = new IngestException( "The supplied descriptive metadata (MODS) does not meet CDR vocabulary requirements."); e.setErrorXML(logger.getAllEvents()); throw e; } // Detect if MODS is present by retrieving it. boolean modsExists = false; try { this.getAccessClient().getDatastreamDissemination(pid, "MD_DESCRIPTIVE", null); modsExists = true; } catch (FedoraException ignored) { } String modsID = "MD_DESCRIPTIVE"; String modsLabel = "Descriptive Metadata (MODS)"; Document modsContent; try { modsContent = new SAXBuilder().build(newMODSFile); } catch (JDOMException e1) { throw new Error("Unexpected JDOM parse exception", e1); } catch (IOException e1) { throw new Error("Unexpected IOException", e1); } try { if (modsExists) { result = this.getManagementClient().modifyInlineXMLDatastream(pid, modsID, false, message, new ArrayList<String>(), modsLabel, modsContent); logger.logEvent(Type.INGESTION, message, pid, modsID); } else { result = this.getManagementClient().addInlineXMLDatastream(pid, modsID, false, message, new ArrayList<String>(), modsLabel, true, modsContent); logger.logEvent(Type.CREATION, message, pid, modsID); } } catch (FedoraException | IOException e) { throw new IngestException("Could not update the specified object.", e); } // update object label based on new MODS title String label = ModsXmlHelper.getFormattedLabelText(modsContent.getRootElement()); if (label != null && label.trim().length() > 0) { try { this.getManagementClient().modifyObject(pid, label, "", State.ACTIVE, message); } catch (FedoraException e) { throw new IngestException("Could not update label for " + pid.getPid(), e); } } // Dublin Core crosswalk Document dc = new Document(); try { dc = ModsXmlHelper.transform(modsContent.getRootElement()); this.getManagementClient().modifyInlineXMLDatastream(pid, "DC", false, message, new ArrayList<String>(), "Internal XML Metadata", dc); String msg = "Metadata Object Description Schema (MODS) data transformed into Dublin Core (DC)."; logger.logDerivationEvent(PremisEventLogger.Type.NORMALIZATION, msg, pid, "MD_DESCRIPTIVE", "DC"); } catch (TransformerException e) { log.error("Cannot cross walk MODS to Dublin Core on update of " + pid.getPid(), e); } catch (FedoraException e) { log.error("Cannot cross walk MODS to Dublin Core on update of " + pid.getPid(), e); } // update PREMIS log try { this.forwardedManagementClient.writePremisEventsToFedoraObject(logger, pid); } catch (FedoraException e) { log.error("Cannot log PREMIS events for " + pid.getPid(), e); } return result; } @Override public String addOrReplaceDatastream(PID pid, Datastream datastream, File content, String mimetype, String user, String message) throws UpdateException { return addOrReplaceDatastream(pid, datastream, null, content, mimetype, user, message); } @Override public String addOrReplaceDatastream(PID pid, Datastream datastream, String label, File content, String mimetype, String user, String message) throws UpdateException { String dsLabel = datastream.getLabel(); if (label != null) dsLabel = label; List<String> datastreamNames = tripleStoreQueryService.listDisseminators(pid); log.debug("Current datastreams: " + datastreamNames); String datastreamName = pid.getURI() + "/" + datastream.getName(); log.debug("Adding or replacing datastream: " + datastreamName); try { if (datastream.getControlGroup().equals(ContentModelHelper.ControlGroup.INTERNAL)) { // Handle inline datastreams if (datastreamNames.contains(datastreamName)) { log.debug("Replacing preexisting internal datastream " + datastreamName); return this.forwardedManagementClient.modifyDatastreamByValue(pid, datastream.getName(), false, message, new ArrayList<String>(), datastream.getLabel(), mimetype, null, null, content); } else { log.debug("Adding internal datastream " + datastreamName); return this.forwardedManagementClient.addInlineXMLDatastream(pid, datastream.getName(), false, message, new ArrayList<String>(), datastream.getLabel(), datastream.isVersionable(), content); } } else if (datastream.getControlGroup().equals(ContentModelHelper.ControlGroup.MANAGED)) { // Handle managed datastreams String dsLocation = forwardedManagementClient.upload(content); if (datastreamNames.contains(datastreamName)) { log.debug("Replacing preexisting managed datastream " + datastreamName); return forwardedManagementClient.modifyDatastreamByReference(pid, datastream.getName(), false, message, Collections.<String> emptyList(), dsLabel, mimetype, null, null, dsLocation); } else { log.debug("Adding managed datastream " + datastreamName); return forwardedManagementClient.addManagedDatastream(pid, datastream.getName(), false, message, Collections.<String> emptyList(), dsLabel, datastream.isVersionable(), mimetype, dsLocation); } } } catch (FedoraException | IOException e) { throw new UpdateException("Failed to modify datastream " + datastream.getName() + " for " + pid.getPid(), e); } return null; } @Override public void move(List<PID> moving, PID destination, String user, String message) throws IngestException { availableCheck(); long startTime = System.currentTimeMillis(); // Verify the destination exists List<PID> destinationPath = this.getTripleStoreQueryService().lookupRepositoryAncestorPids(destination); if (destinationPath == null || destinationPath.size() == 0) { throw new IngestException("Cannot find the destination folder: " + destinationPath); } // Verify the destination is a container List<URI> cmtypes = this.getTripleStoreQueryService().lookupContentModels(destination); if (!cmtypes.contains(ContentModelHelper.Model.CONTAINER.getURI())) { throw new IngestException("The destination is not a folder: " + destinationPath + " " + destination.getPid()); } // Check that none of the items being moved are the destination or one of its ancestors for (PID pid : moving) { if (pid.equals(destination)) throw new IngestException("The destination folder and one of the moving objects are the same: " + destination); for (PID destPid : destinationPath) { if (pid.equals(destPid)) throw new IngestException("The destination folder is below one of the moving objects: " + destination); } } // Determine the set of parents for all of the PIDs to be moved Map<PID, List<PID>> sources = getChildrenContainerMap(moving); // Check that the user has permissions to add/remove from all sources and the destination AccessGroupSet groups = GroupsThreadStore.getGroups(); List<PID> containerList = new ArrayList<>(sources.keySet()); containerList.add(destination); for (PID container : containerList) { if (!aclService.hasAccess(container, groups, Permission.addRemoveContents)) { throw new IngestException("Insufficient permissions to perform move operation", new AuthorizationException("Cannot complete move operation, user " + user + " does not have permission to modify " + container)); } } // Get RELS-EXT documents for the set of parents and replace moved children with tombstones try { for (Entry<PID, List<PID>> sourceEntry : sources.entrySet()) { PID sourcePID = sourceEntry.getKey(); // replace the moved children with tombstones and clear them out of MD_CONTENTS removeChildren(sourcePID, sourceEntry.getValue(), true); } } catch (Exception e) { log.error("Failed to remove children from sources during move, attempting to rollback", e); rollbackMove(sources); throw new IngestException("Failed to remove " + moving.size() + " objects from their source(s)", e); } List<PID> reordered = new ArrayList<>(); // Add the moved children to the destination RELS-EXT try { addChildren(destination, moving, reordered); } catch (Exception e) { // Unexpected failure during move, need to fail operation and roll back log.error("Failed to add children to destination {} during move, attempting to rollback", destination, e); rollbackMove(sources); throw new IngestException("Failed to move " + moving.size() + " objects into " + destination, e); } // Remove tombstones from source containers try { for (Entry<PID, List<PID>> sourceEntry : sources.entrySet()) { cleanupRemovedChildren(sourceEntry.getKey(), sourceEntry.getValue()); } } catch (Exception e) { log.error("Failed to cleanup children tombstones from sources during move", e); rollbackMove(sources); throw new IngestException("Failed to cleanup move of " + moving.size() + " objects to " + destination, e); } log.info("Move operation of {} items to {} completed in {}ms", new Object[] { moving.size(), destination, (System.currentTimeMillis() - startTime) }); // Send out notification message that the move has completed if (this.getOperationsMessageSender() != null) { this.getOperationsMessageSender().sendMoveOperation(user, sources.keySet(), destination, moving, reordered); } } private void rollbackMove(Map<PID, List<PID>> sources) throws IngestException { for (Entry<PID, List<PID>> sourceEntry : sources.entrySet()) { rollbackMove(sourceEntry.getKey(), sourceEntry.getValue()); } } /** * Attempts to rollback a failed move operation by returning part way moved objects to their original source * container and cleaning up removal markers * * @param source * @param moving * @throws IngestException */ @Override public void rollbackMove(PID source, List<PID> moving) throws IngestException { try { DatastreamDocument sourceRelsExtResp; try { sourceRelsExtResp = managementClient.getRELSEXTWithRetries(source); } catch (NotFoundException e) { log.error("Failed to get source RELS-EXT while attempting to roll back move operating from {}", source); return; } Document sourceRelsExt = sourceRelsExtResp.getDocument(); Set<PID> removedChildren = JDOMQueryUtil.getRelationSet(sourceRelsExt.getRootElement(), removedChild); if (removedChildren.size() == 0) { log.debug("No cleanup required for move operation to {}", source); return; } // Clean up the destination(s) // Determine where the children ended up getting moved to Map<PID, List<PID>> destinationMap = getChildrenContainerMap(moving); for (Entry<PID, List<PID>> destEntry : destinationMap.entrySet()) { // Remove all of the moved children from the destination they ended up in removeChildren(destEntry.getKey(), destEntry.getValue(), false); } List<PID> reordered = new ArrayList<>(); // Add the children back to the source addChildren(source, new ArrayList<>(removedChildren), reordered); // Clean up the tombstones cleanupRemovedChildren(source, moving); // Send out notification message that the rollback operation completed if (getOperationsMessageSender() != null) { getOperationsMessageSender().sendMoveOperation("cdr", destinationMap.keySet(), source, moving, reordered); } } catch (FedoraException e) { log.error("Failed to automatically rollback move operation on source {}", source, e); } } /** * Generates a map of children grouped up by common immediate parents * * @param moving * @return */ private Map<PID, List<PID>> getChildrenContainerMap(Collection<PID> moving) { // Determine the set of parents for all of the PIDs to be moved Map<PID, List<PID>> childContainerMap = new HashMap<>(); for (PID pid : moving) { // Get all containers which contain the moved object. String query = String.format( "select $pid from <%1$s> where $pid <%2$s> <%3$s>;", tripleStoreQueryService.getResourceIndexModelUri(), Relationship.contains, pid.getURI()); List<List<String>> result = tripleStoreQueryService.queryResourceIndex(query); if (result == null) { log.warn("Attempting to move orphaned object {}", pid); continue; } for (List<String> sourceList : result) { PID source = new PID(sourceList.get(0)); List<PID> moveFromSource = childContainerMap.get(source); if (moveFromSource == null) { moveFromSource = new ArrayList<>(); childContainerMap.put(source, moveFromSource); } moveFromSource.add(pid); } } return childContainerMap; } /** * Remove children from the provided list within the specified container. If replaceWithMarkers is true, then instead * of removing the relations, they will be replaced with removedChild markers * * @param container * @param children * @param replaceWithMarkers * @throws FedoraException * @throws IngestException */ private void removeChildren(PID container, Collection<PID> children, boolean replaceWithMarkers) throws FedoraException, IngestException { removeRelsExt: do { DatastreamDocument relsExtResp = managementClient.getRELSEXTWithRetries(container); Document relsExt = relsExtResp.getDocument(); try { Element descriptionEl = relsExt.getDocument().getRootElement().getChild("Description", JDOMNamespaceUtil.RDF_NS); List<Element> containsEls = descriptionEl.getChildren(contains.name(), contains.getNamespace()); Iterator<Element> containsIt = containsEls.iterator(); while (containsIt.hasNext()) { Element containsEl = containsIt.next(); PID childPID = new PID(containsEl.getAttributeValue("resource", JDOMNamespaceUtil.RDF_NS)); if (children.contains(childPID)) { if (replaceWithMarkers) { // Switch the moved children to the tombstone relation containsEl.setName(removedChild.name()); } else { // Remove the entry containsIt.remove(); } } } if (log.isDebugEnabled()) { XMLOutputter outputter = new XMLOutputter(org.jdom2.output.Format.getPrettyFormat()); log.debug("Attempting to update RELS-EXT for {} to remove children:\n{}", container, outputter.outputString(relsExt)); } managementClient.modifyDatastream(container, RELS_EXT.getName(), "Removing moved children", relsExtResp.getLastModified(), relsExt); break removeRelsExt; } catch (OptimisticLockException e) { log.debug("Unable to update RELS-EXT for {}, retrying", container, e); } // Repeat rels-ext update if the source changed since the datastream was retrieved } while (true); // Update source MD_CONTENTS to remove children if it is present removeMDContents: do { try { DatastreamDocument mdContents = managementClient.getXMLDatastreamIfExists(container, MD_CONTENTS.getName()); if (mdContents != null) { ContainerContentsHelper.remove(mdContents.getDocument(), children); if (log.isDebugEnabled()) { XMLOutputter outputter = new XMLOutputter(org.jdom2.output.Format.getPrettyFormat()); log.debug("Attempting to update MD_CONTENTS for {} to remove children:\n{}", container, outputter.outputString(mdContents.getDocument())); } managementClient.modifyDatastream(container, MD_CONTENTS.getName(), "Removing " + children.size() + " moved children", mdContents.getLastModified(), mdContents.getDocument()); } break removeMDContents; } catch (OptimisticLockException e) { log.debug("Unable to update MD_CONTENTS for {}, retrying", container, e); } } while (true); } /** * Add a list of children to a container, updating MD_CONTENTS as well if present * * @param container * @param children * @param reordered * @throws FedoraException * @throws IngestException */ private void addChildren(PID container, List<PID> children, Collection<PID> reordered) throws FedoraException, IngestException { updateRelsExt: do { try { DatastreamDocument relsExtResp = managementClient.getRELSEXTWithRetries(container); Document relsExt = relsExtResp.getDocument(); Element descriptionEl = relsExt.getRootElement().getChild("Description", JDOMNamespaceUtil.RDF_NS); // Get the list of existing contains relations to avoid duplicate relations List<Element> containsEls = descriptionEl.getChildren(contains.name(), contains.getNamespace()); Set<PID> existingChildren = new HashSet<>(containsEls.size()); for (Element containsEl : containsEls) { existingChildren.add(new PID(containsEl.getAttributeValue("resource", JDOMNamespaceUtil.RDF_NS))); } // Add children (which are not duplicates) to container for (PID newChild : children) { if (!existingChildren.contains(newChild)) { Element newChildEl = new Element(contains.name(), contains.getNamespace()); newChildEl.setAttribute("resource", newChild.getURI(), JDOMNamespaceUtil.RDF_NS); descriptionEl.addContent(newChildEl); } else { log.warn("Container {} already contained child {}", container, newChild); } } if (log.isDebugEnabled()) { XMLOutputter outputter = new XMLOutputter(org.jdom2.output.Format.getPrettyFormat()); log.debug("Attempting to update RELS-EXT for {} to add children:\n{}", container, outputter.outputString(relsExt)); } // Push changes out to the container container managementClient.modifyDatastream(container, RELS_EXT.getName(), "Adding children", relsExtResp.getLastModified(), relsExt); break updateRelsExt; } catch (OptimisticLockException e) { log.debug("Unable to update RELS-EXT for {}, retrying", container, e); } } while (true); updateMDContents: do { try { // Update MD_CONTENTS to add new children if it is present DatastreamDocument mdContentsResp = managementClient.getXMLDatastreamIfExists(container, MD_CONTENTS.getName()); if (mdContentsResp != null) { Document mdContents = ContainerContentsHelper.addChildContentListInCustomOrder( mdContentsResp.getDocument(), container, children, reordered); if (log.isDebugEnabled()) { XMLOutputter outputter = new XMLOutputter(org.jdom2.output.Format.getPrettyFormat()); log.debug("Attempting to update MD_CONTENTS for {} to add children:\n{}", container, outputter.outputString(mdContents)); } managementClient.modifyDatastream(container, MD_CONTENTS.getName(), "Adding " + children.size() + " children", mdContentsResp.getLastModified(), mdContents); } break updateMDContents; } catch (OptimisticLockException e) { log.debug("Unable to update MD_CONTENTS for {}, retrying", container, e); } } while (true); } /** * Cleanup removedChild references to a list of pids within a particular container. * * @param container * @param children * @throws IngestException * @throws FedoraException */ private void cleanupRemovedChildren(PID container, List<PID> children) throws IngestException, FedoraException { updateRelsExt: do { // Get the current time before accessing RELS-EXT for use in optimistic locking DatastreamDocument relsExtResp = managementClient.getRELSEXTWithRetries(container); Document relsExt = relsExtResp.getDocument(); try { Element descriptionEl = relsExt.getRootElement().getChild("Description", JDOMNamespaceUtil.RDF_NS); List<Element> removedEls = descriptionEl.getChildren(removedChild.name(), removedChild.getNamespace()); Iterator<Element> removedIt = removedEls.iterator(); while (removedIt.hasNext()) { Element removedEl = removedIt.next(); PID childPID = new PID(removedEl.getAttributeValue("resource", JDOMNamespaceUtil.RDF_NS)); // Remove the tombstone if it belongs to this source if (children.contains(childPID)) { removedIt.remove(); } } if (log.isDebugEnabled()) { XMLOutputter outputter = new XMLOutputter(org.jdom2.output.Format.getPrettyFormat()); log.debug("Attempting to update RELS-EXT for {} to clean up children:\n{}", container, outputter.outputString(relsExt)); } managementClient.modifyDatastream(container, RELS_EXT.getName(), "Cleaning up moved children", relsExtResp.getLastModified(), relsExt); break updateRelsExt; } catch (OptimisticLockException e) { log.debug("Unable to update RELS-EXT for {}, retrying", container, e); } // Repeat rels-ext update if the source changed since the datastream was retrieved } while (true); } @Override public void addChildrenToContainer(PID container, List<PID> children) throws FedoraException, IngestException { AccessGroupSet groups = GroupsThreadStore.getGroups(); if (groups != null && !aclService.hasAccess(container, groups, Permission.addRemoveContents)) { throw new AuthorizationException("Insufficient permissions to add children to " + container); } addChildren(container, children, null); } /* * (non-Javadoc) * * @see edu.unc.lib.dl.services.DigitalObjectManager#isAvailable() */ @Override public boolean isAvailable() { return this.available; } public OperationsMessageSender getOperationsMessageSender() { return operationsMessageSender; } public void setOperationsMessageSender(OperationsMessageSender operationsMessageSender) { this.operationsMessageSender = operationsMessageSender; } @Override public PID createContainer(String name, PID parent, Model extraModel, String user, byte[] mods) throws IngestException { PID containerPid = new PID("uuid:"+UUID.randomUUID()); Document foxml = FOXMLJDOMUtil.makeFOXMLDocument(containerPid.getPid()); FOXMLJDOMUtil.setProperty(foxml, ObjectProperty.label, name); PremisEventLogger logger = new PremisEventLogger(user); // MODS if (mods != null) { try(ByteArrayInputStream bais = new ByteArrayInputStream(mods)) { Document modsDoc = new SAXBuilder().build(bais); if(!this.getSchematronValidator().isValid( new JDOMSource(modsDoc), "vocabularies-mods")) { throw new IngestException("MODS was invalid against vocabularies"); } else { Element event = logger.logEvent(Type.VALIDATION, "Validation of Controlled Vocabularies in Descriptive Metadata (MODS)", containerPid, "MD_DESCRIPTIVE"); PremisEventLogger.addDetailedOutcome(event, "MODS is valid", "The supplied MODS metadata meets all CDR vocabulary requirements.", null); } Element modsEl = FOXMLJDOMUtil.makeInlineXMLDatastreamElement( ContentModelHelper.Datastream.MD_DESCRIPTIVE.getName(), ContentModelHelper.Datastream.MD_DESCRIPTIVE.getLabel(), ContentModelHelper.Datastream.MD_DESCRIPTIVE.getName()+"1.0", modsDoc.detachRootElement(), true); foxml.getRootElement().addContent(modsEl); } catch (IOException e) { throw new Error(e); } catch (JDOMException e) { throw new IngestException("MODS records did not parse", e); } } // RELS Element rdfElement = new Element("RDF", JDOMNamespaceUtil.RDF_NS); Element descrElement = new Element("Description", JDOMNamespaceUtil.RDF_NS); descrElement.setAttribute("about", containerPid.getURI(), JDOMNamespaceUtil.RDF_NS); rdfElement.addContent(descrElement); descrElement.addContent( new Element("hasModel", JDOMNamespaceUtil.FEDORA_MODEL_NS).setAttribute( "resource", ContentModelHelper.Model.CONTAINER.getURI().toString(), JDOMNamespaceUtil.RDF_NS)); descrElement.addContent( new Element("hasModel", JDOMNamespaceUtil.FEDORA_MODEL_NS).setAttribute( "resource", ContentModelHelper.Model.PRESERVEDOBJECT.getURI().toString(), JDOMNamespaceUtil.RDF_NS)); if(extraModel != null) { descrElement.addContent( new Element("hasModel", JDOMNamespaceUtil.FEDORA_MODEL_NS).setAttribute( "resource", extraModel.getURI().toString(), JDOMNamespaceUtil.RDF_NS)); } Element relsEl = FOXMLJDOMUtil.makeInlineXMLDatastreamElement( ContentModelHelper.Datastream.RELS_EXT.getName(), ContentModelHelper.Datastream.RELS_EXT.getLabel(), ContentModelHelper.Datastream.RELS_EXT.getName()+"1.0", rdfElement, ContentModelHelper.Datastream.RELS_EXT.isVersionable()); foxml.getRootElement().addContent(relsEl); // PREMIS Element premisEl = new Element("premis", JDOMNamespaceUtil.PREMIS_V2_NS) .addContent(PremisEventLogger.getObjectElement(containerPid)); logger.logEvent(Type.CREATION, "Container created", containerPid); logger.appendLogEvents(containerPid, premisEl); String premisLoc = forwardedManagementClient.upload(new Document(premisEl)); Element premisDSEl = FOXMLJDOMUtil.makeLocatorDatastream( ContentModelHelper.Datastream.MD_EVENTS.getName(), "M", premisLoc, "text/xml", "URL", ContentModelHelper.Datastream.MD_EVENTS.getLabel(), false, null); foxml.getRootElement().addContent(premisDSEl); if(log.isDebugEnabled()) { log.debug(new XMLOutputter().outputString(foxml)); } try { // Add the container to its parent addChildrenToContainer(parent, Arrays.asList(containerPid)); // Ingest the container forwardedManagementClient.ingest(foxml, Format.FOXML_1_1, "Container created via Admin UI"); } catch (FedoraException e) { throw new IngestException("Failed to ingest container object", e); } return containerPid; } public void setCollectionsPid(PID collectionsPid) { this.collectionsPid = collectionsPid; } public FedoraAccessControlService getAclService() { return aclService; } public void setAclService(FedoraAccessControlService aclService) { this.aclService = aclService; } }