/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE file at the root of the source
* tree and available online at
*
* https://github.com/keeps/roda
*/
package org.roda.core.plugins;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.solr.client.solrj.SolrServerException;
import org.roda.core.RodaCoreFactory;
import org.roda.core.TestsHelper;
import org.roda.core.common.iterables.CloseableIterable;
import org.roda.core.common.monitor.TransferredResourcesScanner;
import org.roda.core.data.common.RodaConstants;
import org.roda.core.data.common.RodaConstants.SolrType;
import org.roda.core.data.exceptions.AlreadyExistsException;
import org.roda.core.data.exceptions.AuthorizationDeniedException;
import org.roda.core.data.exceptions.GenericException;
import org.roda.core.data.exceptions.InvalidParameterException;
import org.roda.core.data.exceptions.JobAlreadyStartedException;
import org.roda.core.data.exceptions.NotFoundException;
import org.roda.core.data.exceptions.RequestNotValidException;
import org.roda.core.data.v2.common.OptionalWithCause;
import org.roda.core.data.v2.index.select.SelectedItemsAll;
import org.roda.core.data.v2.index.select.SelectedItemsList;
import org.roda.core.data.v2.ip.AIP;
import org.roda.core.data.v2.ip.File;
import org.roda.core.data.v2.ip.Representation;
import org.roda.core.data.v2.ip.TransferredResource;
import org.roda.core.data.v2.jobs.Job;
import org.roda.core.data.v2.jobs.PluginType;
import org.roda.core.index.IndexService;
import org.roda.core.model.ModelService;
import org.roda.core.model.utils.ModelUtils;
import org.roda.core.plugins.orchestrate.JobsHelper;
import org.roda.core.plugins.plugins.base.ReplicationPlugin;
import org.roda.core.plugins.plugins.ingest.TransferredResourceToAIPPlugin;
import org.roda.core.storage.Binary;
import org.roda.core.storage.StorageService;
import org.roda.core.storage.fs.FSUtils;
import org.roda.core.util.CommandException;
import org.roda.core.util.CommandUtility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.AssertJUnit;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@Test(groups = {RodaConstants.TEST_GROUP_ALL, RodaConstants.TEST_GROUP_TRAVIS})
public class ReplicationPluginTest {
private static final Logger LOGGER = LoggerFactory.getLogger(ReplicationPluginTest.class);
private static final int GENERATED_FILE_SIZE = 100;
private static final int NUMBER_OF_AIPS = 1000;
private static Path basePath;
private static Path testPath;
private static StorageService storage;
private static ModelService model;
private static IndexService index;
private static String parentResourceUUID = null;
private static int resourceCounter = 0;
@BeforeClass
public void setUp() throws Exception {
basePath = TestsHelper.createBaseTempDir(getClass(), true,
PosixFilePermissions
.asFileAttribute(new HashSet<>(Arrays.asList(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE,
PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_EXECUTE))));
boolean deploySolr = true;
boolean deployLdap = true;
boolean deployFolderMonitor = true;
boolean deployOrchestrator = true;
boolean deployPluginManager = true;
boolean deployDefaultResources = false;
// embedded Apache Solr
RodaCoreFactory.instantiateTest(deploySolr, deployLdap, deployFolderMonitor, deployOrchestrator,
deployPluginManager, deployDefaultResources, SolrType.EMBEDDED);
// // HTTP Apache Solr
// RodaCoreFactory.instantiateTest(deploySolr, deployLdap,
// deployFolderMonitor, deployOrchestrator,
// deployPluginManager, deployDefaultResources, SolrType.HTTP);
// // Cloud Apache Solr
// RodaCoreFactory.instantiateTest(deploySolr, deployLdap,
// deployFolderMonitor, deployOrchestrator,
// deployPluginManager, deployDefaultResources, SolrType.HTTP_CLOUD);
model = RodaCoreFactory.getModelService();
index = RodaCoreFactory.getIndexService();
storage = RodaCoreFactory.getStorageService();
testPath = TestsHelper.createBaseTempDir(getClass(), true,
PosixFilePermissions
.asFileAttribute(new HashSet<>(Arrays.asList(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE,
PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_EXECUTE))));
Files.createDirectories(
testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER).resolve(RodaConstants.STORAGE_CONTAINER_AIP));
Files.createDirectories(
testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER).resolve(RodaConstants.STORAGE_CONTAINER_PRESERVATION)
.resolve(RodaConstants.STORAGE_CONTAINER_PRESERVATION_AGENTS));
LOGGER.info("Running {} tests under storage {}", getClass().getSimpleName(), basePath);
}
@AfterClass
public void tearDown() throws Exception {
RodaCoreFactory.shutdown();
FSUtils.deletePath(basePath);
FSUtils.deletePath(testPath);
}
private ByteArrayInputStream generateContentData() {
return new ByteArrayInputStream(RandomStringUtils.randomAscii(GENERATED_FILE_SIZE).getBytes());
}
private String createCorpora() throws InterruptedException, IOException, NotFoundException, GenericException,
RequestNotValidException, AlreadyExistsException {
TransferredResourcesScanner f = RodaCoreFactory.getTransferredResourcesScanner();
String parentUUID = f.createFolder(null, "test").getUUID();
index.commit(TransferredResource.class);
index.retrieve(TransferredResource.class, parentUUID, new ArrayList<>());
return parentUUID;
}
private TransferredResource createCorpora(String parentUUID, int i) throws InterruptedException, IOException,
NotFoundException, GenericException, RequestNotValidException, AlreadyExistsException {
TransferredResourcesScanner f = RodaCoreFactory.getTransferredResourcesScanner();
String test1UUID = f.createFolder(parentUUID, "test" + i).getUUID();
f.createFile(test1UUID, "test" + i + ".txt", generateContentData());
index.commit(TransferredResource.class);
return index.retrieve(TransferredResource.class, test1UUID, new ArrayList<>());
}
private List<String> ingestCorpora()
throws RequestNotValidException, NotFoundException, GenericException, AlreadyExistsException,
AuthorizationDeniedException, InvalidParameterException, InterruptedException, IOException, SolrServerException {
if (parentResourceUUID == null) {
parentResourceUUID = createCorpora();
}
List<String> resourceUUIDs = new ArrayList<>();
for (int i = 0; i < NUMBER_OF_AIPS / 2; i++) {
TransferredResource transferredResource = createCorpora(parentResourceUUID, resourceCounter);
AssertJUnit.assertNotNull(transferredResource);
resourceUUIDs.add(transferredResource.getUUID());
resourceCounter++;
}
Job job = TestsHelper.executeJob(TransferredResourceToAIPPlugin.class, new HashMap<>(), PluginType.SIP_TO_AIP,
SelectedItemsList.create(TransferredResource.class, resourceUUIDs));
index.commitAIPs();
return TestsHelper.getJobReports(index, job).stream().map(r -> r.getOutcomeObjectId()).collect(Collectors.toList());
}
@Test
private void testRsyncViaListOfIds() throws RequestNotValidException, NotFoundException, GenericException,
AlreadyExistsException, AuthorizationDeniedException, InvalidParameterException, InterruptedException, IOException,
SolrServerException, JobAlreadyStartedException, CommandException {
// increase number of job workers
int originalNumberOfJobWorkers = JobsHelper.getNumberOfJobsWorkers();
JobsHelper.setNumberOfJobsWorkers(originalNumberOfJobWorkers * 2);
// increase sync job execution timeout in seconds
JobsHelper.setSyncTimeout(1200);
LOGGER.info("Ingesting first pack of corpora...");
List<String> aips = ingestCorpora();
LOGGER.info("Rsyncing first pack of corpora...");
executeJobWithReplicationPlugin(aips);
LOGGER.info("Ingesting second pack of corpora...");
List<String> newAips = ingestCorpora();
LOGGER.info("Rsyncing second pack of corpora...");
executeJobWithReplicationPlugin(newAips);
LOGGER.info("Testing if destination has the number of AIPs that was supposed");
Assert.assertEquals(FSUtils
.countPath(testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER).resolve(RodaConstants.STORAGE_CONTAINER_AIP))
.intValue(), NUMBER_OF_AIPS);
AIP aip1 = model.retrieveAIP(aips.get(0));
AIP aip2 = model.retrieveAIP(aips.get(1));
Binary binary = null;
Representation rep1 = aip1.getRepresentations().get(0);
CloseableIterable<OptionalWithCause<File>> listFilesUnder1 = model.listFilesUnder(aip1.getId(), rep1.getId(),
false);
Iterator<OptionalWithCause<File>> iterator1 = listFilesUnder1.iterator();
while (iterator1.hasNext()) {
OptionalWithCause<File> next = iterator1.next();
File file = next.get();
binary = storage.getBinary(
ModelUtils.getFileStoragePath(file.getAipId(), file.getRepresentationId(), file.getPath(), file.getId()));
}
Representation rep2 = aip2.getRepresentations().get(0);
CloseableIterable<OptionalWithCause<File>> listFilesUnder2 = model.listFilesUnder(aip2.getId(), rep2.getId(),
false);
Iterator<OptionalWithCause<File>> iterator2 = listFilesUnder2.iterator();
while (iterator2.hasNext()) {
OptionalWithCause<File> next = iterator2.next();
File file = next.get();
if (binary != null) {
model.updateFile(file.getAipId(), file.getRepresentationId(), file.getPath(), file.getId(), binary.getContent(),
true, true);
}
}
executeJobWithReplicationPlugin(Arrays.asList(aip2.getId()));
List<String> diffCommand = new ArrayList<>();
diffCommand.add("diff");
diffCommand.add("-rq");
diffCommand.add(RodaCoreFactory.getDataPath().resolve(RodaConstants.CORE_STORAGE_FOLDER)
.resolve(RodaConstants.STORAGE_CONTAINER_AIP).toString());
diffCommand
.add(testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER).resolve(RodaConstants.STORAGE_CONTAINER_AIP).toString());
LOGGER.info("Executing diff to compare two folders: {}", diffCommand);
CommandUtility.execute(diffCommand);
}
@Test(enabled = false)
private void testRsyncViaFilter() throws RequestNotValidException, NotFoundException, GenericException,
AlreadyExistsException, AuthorizationDeniedException, InvalidParameterException, InterruptedException, IOException,
SolrServerException, JobAlreadyStartedException, CommandException {
// increase sync job execution timeout in seconds
JobsHelper.setSyncTimeout(1200);
JobsHelper.setBlockSize(100);
LOGGER.info("Ingesting first pack of corpora...");
ingestCorpora();
LOGGER.info("Ingesting second pack of corpora...");
ingestCorpora();
LOGGER.info("Rsyncing corpora via filter...");
executeJobWithReplicationPlugin();
LOGGER.info("Testing if destination has the number of AIPs that was supposed");
Assert.assertEquals(FSUtils
.countPath(testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER).resolve(RodaConstants.STORAGE_CONTAINER_AIP))
.intValue(), NUMBER_OF_AIPS);
List<String> diffCommand = new ArrayList<>();
diffCommand.add("diff");
diffCommand.add("-rq");
diffCommand.add(RodaCoreFactory.getDataPath().resolve(RodaConstants.CORE_STORAGE_FOLDER)
.resolve(RodaConstants.STORAGE_CONTAINER_AIP).toString());
diffCommand
.add(testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER).resolve(RodaConstants.STORAGE_CONTAINER_AIP).toString());
LOGGER.info("Executing diff to compare two folders: {}", diffCommand);
CommandUtility.execute(diffCommand);
}
private void executeJobWithReplicationPlugin(List<String> aipIds)
throws CommandException, IOException, JobAlreadyStartedException, GenericException, RequestNotValidException,
NotFoundException, AuthorizationDeniedException {
String targetUser = RodaCoreFactory.getRodaConfigurationAsString("core", "aip_rsync", "target");
RodaCoreFactory.getRodaConfiguration().setProperty("core.aip_rsync.target", testPath.toString() + "/");
Job job = TestsHelper.executeJob(ReplicationPlugin.class, new HashMap<>(), PluginType.MISC,
SelectedItemsList.create(AIP.class, aipIds));
TestsHelper.getJobReports(index, job, true);
RodaCoreFactory.getRodaConfiguration().setProperty("core.aip_rsync.target", targetUser);
}
private void executeJobWithReplicationPlugin() throws CommandException, IOException, JobAlreadyStartedException,
GenericException, RequestNotValidException, NotFoundException, AuthorizationDeniedException {
String targetUser = RodaCoreFactory.getRodaConfigurationAsString("core", "aip_rsync", "target");
RodaCoreFactory.getRodaConfiguration().setProperty("core.aip_rsync.target", testPath.toString());
TestsHelper.executeJob(ReplicationPlugin.class, PluginType.MISC, SelectedItemsAll.create(AIP.class));
RodaCoreFactory.getRodaConfiguration().setProperty("core.aip_rsync.target", targetUser);
}
}