/*
* Copyright 2015-2016 OpenCB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opencb.opencga.catalog.utils;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.datastore.core.QueryResult;
import org.opencb.opencga.catalog.CatalogManagerExternalResource;
import org.opencb.opencga.catalog.CatalogManagerTest;
import org.opencb.opencga.catalog.db.api.FileDBAdaptor;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
import org.opencb.opencga.catalog.models.File;
import org.opencb.opencga.catalog.models.Project;
import org.opencb.opencga.catalog.models.Status;
import org.opencb.opencga.catalog.models.Study;
import org.opencb.opencga.core.common.IOUtils;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import static org.junit.Assert.*;
public class FileScannerTest {
@Rule
public CatalogManagerExternalResource catalogManagerExternalResource = new CatalogManagerExternalResource();
public static final String PASSWORD = "asdf";
private CatalogManager catalogManager;
private String sessionIdUser;
private File folder;
private Study study;
private Project project;
private Path directory;
@Before
public void setUp() throws IOException, CatalogException {
catalogManager = catalogManagerExternalResource.getCatalogManager();
catalogManager.createUser("user", "User Name", "mail@ebi.ac.uk", PASSWORD, "", null, null);
sessionIdUser = catalogManager.login("user", PASSWORD, "127.0.0.1").first().getId();
project = catalogManager.getProjectManager().create("Project about some genomes", "1000G", "", "ACME", "Homo sapiens",
null, null, "GRCh38", new QueryOptions(), sessionIdUser).first();
study = catalogManager.createStudy(project.getId(), "Phase 1", "phase1", Study.Type.TRIO, "Done", sessionIdUser).first();
folder = catalogManager.getFileManager().createFolder(Long.toString(study.getId()), Paths.get("data/test/folder/").toString(),
null, true, null, QueryOptions.empty(), sessionIdUser).first();
directory = catalogManagerExternalResource.getOpencgaHome().resolve("catalog_scan_test_folder").toAbsolutePath();
if (directory.toFile().exists()) {
IOUtils.deleteDirectory(directory);
}
Files.createDirectory(directory);
}
@Test
public void testScan() throws IOException, CatalogException {
Files.createDirectory(directory.resolve("subfolder"));
Files.createDirectory(directory.resolve("subfolder/subsubfolder"));
CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString());
CatalogManagerTest.createDebugFile(directory.resolve("file2.txt").toString());
CatalogManagerTest.createDebugFile(directory.resolve("file3.txt").toString());
CatalogManagerTest.createDebugFile(directory.resolve("subfolder/file1.txt").toString());
CatalogManagerTest.createDebugFile(directory.resolve("subfolder/file2.txt").toString());
CatalogManagerTest.createDebugFile(directory.resolve("subfolder/file3.txt").toString());
CatalogManagerTest.createDebugFile(directory.resolve("subfolder/subsubfolder/file1.txt").toString());
CatalogManagerTest.createDebugFile(directory.resolve("subfolder/subsubfolder/file2.txt").toString());
CatalogManagerTest.createDebugFile(directory.resolve("subfolder/subsubfolder/file3.txt").toString());
FileScanner fileScanner = new FileScanner(catalogManager);
List<File> files = fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, true, true, sessionIdUser);
assertEquals(9, files.size());
files.forEach((File file) -> assertTrue(file.getAttributes().containsKey("checksum")));
}
@Test
public void testDeleteExisting() throws IOException, CatalogException {
File file = catalogManager.createFile(study.getId(), File.Format.PLAIN, File.Bioformat.NONE, folder.getPath() + "file1.txt",
CatalogManagerTest.createDebugFile().toURI(), "", false, sessionIdUser).first();
CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString());
List<File> files = new FileScanner(catalogManager).scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, false,
true, sessionIdUser);
files.forEach((File f) -> assertFalse(f.getAttributes().containsKey("checksum")));
assertEquals(File.FileStatus.DELETED, getFile(file.getId()).getStatus().getName());
}
public File getFile(long id) throws CatalogException {
return catalogManager.searchFile(study.getId(),
new Query(FileDBAdaptor.QueryParams.ID.key(), id)
.append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), Status.DELETED + "," + Status.TRASHED + "," + Status.READY), sessionIdUser)
.first();
}
@Test
public void testDeleteTrashed() throws IOException, CatalogException {
File file = catalogManager.createFile(study.getId(), File.Format.PLAIN, File.Bioformat.NONE, folder.getPath() + "file1.txt",
CatalogManagerTest.createDebugFile().toURI(), "", false, sessionIdUser).first();
catalogManager.getFileManager().delete(Long.toString(file.getId()), null, new QueryOptions(), sessionIdUser);
QueryResult<File> fileQueryResult = catalogManager.getFileManager().get(new Query()
.append(FileDBAdaptor.QueryParams.ID.key(), file.getId())
.append(FileDBAdaptor.QueryParams.STUDY_ID.key(), study.getId())
.append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), "!=EMPTY"),
new QueryOptions(), sessionIdUser);
file = fileQueryResult.first();
assertEquals(File.FileStatus.TRASHED, file.getStatus().getName());
// Files.delete(Paths.get(catalogManager.getFileUri(file)));
// List<File> files = new FileScanner(catalogManager).checkStudyFiles(study, false, sessionIdUser);
//
// file = getFile(file.getId());
// assertEquals(File.FileStatus.TRASHED, file.getStatus().getName());
// assertEquals(1, files.size());
// assertEquals(file.getId(), files.get(0).getId());
}
@Test
public void testReplaceExisting() throws IOException, CatalogException {
CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString());
Files.createDirectory(directory.resolve("s/"));
CatalogManagerTest.createDebugFile(directory.resolve("s/file2.txt").toString());
File file = catalogManager.createFile(study.getId(), File.Format.PLAIN, File.Bioformat.NONE, folder.getPath() + "file1.txt",
CatalogManagerTest.createDebugFile().toURI(), "", false, sessionIdUser).first();
catalogManager.createFile(study.getId(), File.Format.PLAIN, File.Bioformat.NONE, folder.getPath() + "s/file2.txt",
CatalogManagerTest.createDebugFile().toURI(), "", true, sessionIdUser).first();
FileScanner fileScanner = new FileScanner(catalogManager);
fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser);
File replacedFile = catalogManager.getFile(file.getId(), sessionIdUser).first();
assertEquals(File.FileStatus.READY, replacedFile.getStatus().getName());
assertEquals(file.getId(), replacedFile.getId());
assertFalse(replacedFile.getAttributes().get("checksum").equals(file.getAttributes().get("checksum")));
}
@Test
public void testRegisterFiles() throws IOException, CatalogException {
Path file1 = directory.resolve("file1.txt");
Path file2 = directory.resolve("s/file2.txt");
Path folder = directory.resolve("s/");
Path file3 = directory.resolve("file3.txt");
CatalogManagerTest.createDebugFile(file1.toString());
Files.createDirectory(folder);
CatalogManagerTest.createDebugFile(file2.toString());
CatalogManagerTest.createDebugFile(file3.toString());
List<Path> filePaths = new ArrayList<>(2);
filePaths.add(file1);
filePaths.add(file2);
FileScanner fileScanner = new FileScanner(catalogManager);
// List<File> files = fileScanner.registerFiles(this.folder, filePaths, FileScanner.FileScannerPolicy.DELETE, true, false, sessionIdUser);
Predicate<URI> uriPredicate = uri -> uri.getPath().endsWith("file1.txt") || uri.getPath().endsWith("file2.txt");
List<File> files = fileScanner.scan(this.folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, true, false, uriPredicate, -1, sessionIdUser);
assertEquals(2, files.size());
for (File file : files) {
assertTrue(Paths.get(file.getUri()).toFile().exists());
}
for (Path filePath : filePaths) {
assertTrue(filePath.toFile().exists());
}
assertTrue(file3.toFile().exists());
}
@Test
public void testScanStudyURI() throws IOException, CatalogException {
CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString());
FileScanner fileScanner = new FileScanner(catalogManager);
List<File> files = fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser);
assertEquals(1, files.size());
URI studyUri = catalogManager.getStudyUri(study.getId());
CatalogManagerTest.createDebugFile(studyUri.resolve("data/test/folder/").resolve("file2.txt").getPath());
File root = catalogManager.searchFile(study.getId(), new Query("name", "."), sessionIdUser).first();
files = fileScanner.scan(root, studyUri, FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser);
assertEquals(1, files.size());
files.forEach((f) -> assertTrue(f.getSize() > 0));
files.forEach((f) -> assertEquals(f.getStatus().getName(), File.FileStatus.READY));
files.forEach((f) -> assertTrue(f.getAttributes().containsKey("checksum")));
}
@Test
public void testResyncStudy() throws IOException, CatalogException {
CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString());
//ReSync study folder. Will detect any difference.
FileScanner fileScanner = new FileScanner(catalogManager);
List<File> files;
files = fileScanner.reSync(study, true, sessionIdUser);
assertEquals(0, files.size());
//Add one extra file. ReSync study folder.
URI studyUri = catalogManager.getStudyUri(study.getId());
Path filePath = CatalogManagerTest.createDebugFile(studyUri.resolve("data/test/folder/").resolve("file_scanner_test_file.txt").getPath()).toPath();
files = fileScanner.reSync(study, true, sessionIdUser);
assertEquals(1, files.size());
File file = files.get(0);
assertTrue(file.getSize() > 0);
assertEquals(File.FileStatus.READY, file.getStatus().getName());
assertTrue(file.getAttributes().containsKey("checksum"));
//Delete file. CheckStudyFiles. Will detect one File.Status.MISSING file
Files.delete(filePath);
files = fileScanner.checkStudyFiles(study, true, sessionIdUser);
assertEquals(1, files.size());
assertEquals(File.FileStatus.MISSING, files.get(0).getStatus().getName());
String originalChecksum = files.get(0).getAttributes().get("checksum").toString();
//Restore file. CheckStudyFiles. Will detect one re-tracked file. Checksum must be different.
CatalogManagerTest.createDebugFile(filePath.toString());
files = fileScanner.checkStudyFiles(study, true, sessionIdUser);
assertEquals(1, files.size());
assertEquals(File.FileStatus.READY, files.get(0).getStatus().getName());
String newChecksum = files.get(0).getAttributes().get("checksum").toString();
assertFalse(originalChecksum.equals(newChecksum));
//Delete file. ReSync. Will detect one File.Status.MISSING file (like checkFile)
Files.delete(filePath);
files = fileScanner.reSync(study, true, sessionIdUser);
assertEquals(1, files.size());
assertEquals(File.FileStatus.MISSING, files.get(0).getStatus().getName());
originalChecksum = files.get(0).getAttributes().get("checksum").toString();
//Restore file. CheckStudyFiles. Will detect one found file. Checksum must be different.
CatalogManagerTest.createDebugFile(filePath.toString());
files = fileScanner.reSync(study, true, sessionIdUser);
assertEquals(1, files.size());
assertEquals(File.FileStatus.READY, files.get(0).getStatus().getName());
newChecksum = files.get(0).getAttributes().get("checksum").toString();
assertFalse(originalChecksum.equals(newChecksum));
}
@Test
public void testComplexAdd() throws IOException, CatalogException {
CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.gz").toString());
CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json").toString());
CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json.gz").toString());
CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json.snappy").toString());
CatalogManagerTest.createDebugFile(directory.resolve("file2.bam").toString());
CatalogManagerTest.createDebugFile(directory.resolve("file2.sam.gz").toString());
FileScanner fileScanner = new FileScanner(catalogManager);
List<File> files = fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser);
Map<String, File> map = files.stream().collect(Collectors.toMap(File::getName, (f) -> f));
assertEquals(6, files.size());
files.forEach((file) -> assertEquals(File.FileStatus.READY, file.getStatus().getName()));
assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.gz").getBioformat());
assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.variants.json").getBioformat());
assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.variants.json.gz").getBioformat());
assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.variants.json.snappy").getBioformat());
assertEquals(File.Bioformat.ALIGNMENT, map.get("file2.bam").getBioformat());
assertEquals(File.Bioformat.ALIGNMENT, map.get("file2.sam.gz").getBioformat());
assertEquals(File.Format.VCF, map.get("file1.vcf.gz").getFormat());
assertEquals(File.Format.JSON, map.get("file1.vcf.variants.json").getFormat());
assertEquals(File.Format.JSON, map.get("file1.vcf.variants.json.gz").getFormat());
assertEquals(File.Format.JSON, map.get("file1.vcf.variants.json.snappy").getFormat());
assertEquals(File.Format.BAM, map.get("file2.bam").getFormat());
assertEquals(File.Format.SAM, map.get("file2.sam.gz").getFormat());
}
}