/*
* The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
* (the "License"). You may not use this work except in compliance with the License, which is
* available at www.apache.org/licenses/LICENSE-2.0
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied, as more fully set forth in the License.
*
* See the NOTICE file distributed with this work for information regarding copyright ownership.
*/
package alluxio.client;
import alluxio.AlluxioURI;
import alluxio.LocalAlluxioClusterResource;
import alluxio.BaseIntegrationTest;
import alluxio.client.file.FileInStream;
import alluxio.client.file.FileSystem;
import alluxio.client.file.options.CreateFileOptions;
import alluxio.client.file.options.MountOptions;
import alluxio.master.LocalAlluxioCluster;
import alluxio.master.MasterRegistry;
import alluxio.master.MasterTestUtils;
import alluxio.master.file.FileSystemMaster;
import alluxio.underfs.ConfExpectingUnderFileSystemFactory;
import alluxio.underfs.UnderFileSystem;
import alluxio.underfs.UnderFileSystemConfiguration;
import alluxio.underfs.UnderFileSystemFactoryRegistry;
import alluxio.underfs.local.LocalUnderFileSystemFactory;
import alluxio.util.UnderFileSystemUtils;
import alluxio.util.io.PathUtils;
import alluxio.wire.MountPointInfo;
import com.google.common.collect.ImmutableMap;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.util.Map;
/**
* Integration tests for mounting multiple UFSes into Alluxio, each with a different configuration.
*/
public final class MultiUfsMountIntegrationTest extends BaseIntegrationTest {
private static final String MOUNT_POINT1 = "/mnt1";
private static final String MOUNT_POINT2 = "/mnt2";
private static final Map<String, String> UFS_CONF1 = ImmutableMap.of("key1", "val1");
private static final Map<String, String> UFS_CONF2 = ImmutableMap.of("key2", "val2");
private ConfExpectingUnderFileSystemFactory mUfsFactory1;
private ConfExpectingUnderFileSystemFactory mUfsFactory2;
private AlluxioURI mMountPoint1 = new AlluxioURI(MOUNT_POINT1);
private AlluxioURI mMountPoint2 = new AlluxioURI(MOUNT_POINT2);
private String mUfsUri1;
private String mUfsUri2;
private UnderFileSystem mLocalUfs;
private FileSystem mFileSystem;
private LocalAlluxioCluster mLocalAlluxioCluster;
@Rule
public TemporaryFolder mFolder = new TemporaryFolder();
@Rule
public LocalAlluxioClusterResource mLocalAlluxioClusterResource =
new LocalAlluxioClusterResource.Builder().setStartCluster(false).build();
@Before
public void before() throws Exception {
mUfsFactory1 = new ConfExpectingUnderFileSystemFactory("ufs1", UFS_CONF1);
mUfsFactory2 = new ConfExpectingUnderFileSystemFactory("ufs2", UFS_CONF2);
UnderFileSystemFactoryRegistry.register(mUfsFactory1);
UnderFileSystemFactoryRegistry.register(mUfsFactory2);
mUfsUri1 = "ufs1://" + mFolder.newFolder().getAbsoluteFile();
mUfsUri2 = "ufs2://" + mFolder.newFolder().getAbsoluteFile();
mLocalUfs = new LocalUnderFileSystemFactory().create(mFolder.getRoot().getAbsolutePath(),
UnderFileSystemConfiguration.defaults());
mLocalAlluxioClusterResource.start();
mLocalAlluxioCluster = mLocalAlluxioClusterResource.get();
mFileSystem = mLocalAlluxioCluster.getClient();
// Mount ufs1 to /mnt1 with specified options.
MountOptions options1 =
MountOptions.defaults().setProperties(UFS_CONF1);
mFileSystem.mount(mMountPoint1, new AlluxioURI(mUfsUri1), options1);
// Mount ufs2 to /mnt2 with specified options.
MountOptions options2 =
MountOptions.defaults().setProperties(UFS_CONF2);
mFileSystem.mount(mMountPoint2, new AlluxioURI(mUfsUri2), options2);
}
@After
public void after() throws Exception {
UnderFileSystemFactoryRegistry.unregister(mUfsFactory1);
UnderFileSystemFactoryRegistry.unregister(mUfsFactory2);
}
@Test
public void createFile() throws Exception {
CreateFileOptions writeBoth =
CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH);
AlluxioURI file1 = mMountPoint1.join("file1");
AlluxioURI file2 = mMountPoint2.join("file2");
mFileSystem.createFile(file1, writeBoth).close();
mFileSystem.createFile(file2, writeBoth).close();
Assert.assertTrue(mLocalUfs.isFile(PathUtils.concatPath(mUfsUri1, "file1")));
Assert.assertTrue(mLocalUfs.isFile(PathUtils.concatPath(mUfsUri2, "file2")));
}
@Test
public void createDirectory() throws Exception {
AlluxioURI dir1 = mMountPoint1.join("dir1");
AlluxioURI dir2 = mMountPoint2.join("dir2");
mFileSystem.createDirectory(dir1);
mFileSystem.createDirectory(dir2);
Assert.assertTrue(mLocalUfs.isDirectory(PathUtils.concatPath(mUfsUri1, "dir1")));
Assert.assertTrue(mLocalUfs.isDirectory(PathUtils.concatPath(mUfsUri2, "dir2")));
}
@Test
public void deleteFile() throws Exception {
String ufsFile1 = PathUtils.concatPath(mUfsUri1, "file1");
String ufsFile2 = PathUtils.concatPath(mUfsUri2, "file2");
UnderFileSystemUtils.touch(mLocalUfs, ufsFile1);
UnderFileSystemUtils.touch(mLocalUfs, ufsFile2);
AlluxioURI file1 = mMountPoint1.join("file1");
AlluxioURI file2 = mMountPoint2.join("file2");
Assert.assertTrue(mFileSystem.exists(file1));
Assert.assertTrue(mFileSystem.exists(file2));
mFileSystem.delete(file1);
mFileSystem.delete(file2);
Assert.assertFalse(mFileSystem.exists(file1));
Assert.assertFalse(mFileSystem.exists(file2));
Assert.assertFalse(mLocalUfs.exists(ufsFile1));
Assert.assertFalse(mLocalUfs.exists(ufsFile2));
}
@Test
public void deleteDirectory() throws Exception {
String ufsDir1 = PathUtils.concatPath(mUfsUri1, "dir1");
String ufsDir2 = PathUtils.concatPath(mUfsUri2, "dir2");
UnderFileSystemUtils.mkdirIfNotExists(mLocalUfs, ufsDir1);
UnderFileSystemUtils.mkdirIfNotExists(mLocalUfs, ufsDir2);
AlluxioURI dir1 = mMountPoint1.join("dir1");
AlluxioURI dir2 = mMountPoint2.join("dir2");
Assert.assertTrue(mFileSystem.exists(dir1));
Assert.assertTrue(mFileSystem.exists(dir2));
mFileSystem.delete(dir1);
mFileSystem.delete(dir2);
Assert.assertFalse(mFileSystem.exists(dir1));
Assert.assertFalse(mFileSystem.exists(dir2));
}
@Test
public void renameFile() throws Exception {
String ufsFileSrc1 = PathUtils.concatPath(mUfsUri1, "file1");
String ufsFileSrc2 = PathUtils.concatPath(mUfsUri2, "file2");
String ufsFileDst1 = PathUtils.concatPath(mUfsUri1, "renamedFile1");
String ufsFileDst2 = PathUtils.concatPath(mUfsUri2, "renamedFile2");
UnderFileSystemUtils.touch(mLocalUfs, ufsFileSrc1);
UnderFileSystemUtils.touch(mLocalUfs, ufsFileSrc2);
AlluxioURI fileSrc1 = mMountPoint1.join("file1");
AlluxioURI fileSrc2 = mMountPoint2.join("file2");
AlluxioURI fileDst1 = mMountPoint1.join("renamedFile1");
AlluxioURI fileDst2 = mMountPoint2.join("renamedFile2");
Assert.assertTrue(mFileSystem.exists(fileSrc1));
Assert.assertTrue(mFileSystem.exists(fileSrc2));
mFileSystem.rename(fileSrc1, fileDst1);
mFileSystem.rename(fileSrc2, fileDst2);
Assert.assertFalse(mFileSystem.exists(fileSrc1));
Assert.assertFalse(mFileSystem.exists(fileSrc2));
Assert.assertTrue(mFileSystem.exists(fileDst1));
Assert.assertTrue(mFileSystem.exists(fileDst2));
Assert.assertFalse(mLocalUfs.exists(ufsFileSrc1));
Assert.assertFalse(mLocalUfs.exists(ufsFileSrc2));
Assert.assertTrue(mLocalUfs.exists(ufsFileDst1));
Assert.assertTrue(mLocalUfs.exists(ufsFileDst2));
}
@Test
public void renameDirectory() throws Exception {
String ufsDirSrc1 = PathUtils.concatPath(mUfsUri1, "dir1");
String ufsDirSrc2 = PathUtils.concatPath(mUfsUri2, "dir2");
String ufsDirDst1 = PathUtils.concatPath(mUfsUri1, "renamedDir1");
String ufsDirDst2 = PathUtils.concatPath(mUfsUri2, "renamedDir2");
UnderFileSystemUtils.touch(mLocalUfs, ufsDirSrc1);
UnderFileSystemUtils.touch(mLocalUfs, ufsDirSrc2);
AlluxioURI dirSrc1 = mMountPoint1.join("dir1");
AlluxioURI dirSrc2 = mMountPoint2.join("dir2");
AlluxioURI dirDst1 = mMountPoint1.join("renamedDir1");
AlluxioURI dirDst2 = mMountPoint2.join("renamedDir2");
Assert.assertTrue(mFileSystem.exists(dirSrc1));
Assert.assertTrue(mFileSystem.exists(dirSrc2));
mFileSystem.rename(dirSrc1, dirDst1);
mFileSystem.rename(dirSrc2, dirDst2);
Assert.assertFalse(mFileSystem.exists(dirSrc1));
Assert.assertFalse(mFileSystem.exists(dirSrc2));
Assert.assertTrue(mFileSystem.exists(dirDst1));
Assert.assertTrue(mFileSystem.exists(dirDst2));
Assert.assertFalse(mLocalUfs.exists(ufsDirSrc1));
Assert.assertFalse(mLocalUfs.exists(ufsDirSrc2));
Assert.assertTrue(mLocalUfs.exists(ufsDirDst1));
Assert.assertTrue(mLocalUfs.exists(ufsDirDst2));
}
@Test
public void openFile() throws Exception {
String ufsFile1 = PathUtils.concatPath(mUfsUri1, "file1");
String ufsFile2 = PathUtils.concatPath(mUfsUri2, "file2");
UnderFileSystemUtils.touch(mLocalUfs, ufsFile1);
UnderFileSystemUtils.touch(mLocalUfs, ufsFile2);
AlluxioURI file1 = mMountPoint1.join("file1");
AlluxioURI file2 = mMountPoint2.join("file2");
Assert.assertTrue(mFileSystem.exists(file1));
Assert.assertTrue(mFileSystem.exists(file2));
FileInStream inStream1 = mFileSystem.openFile(file1);
Assert.assertNotNull(inStream1);
inStream1.close();
FileInStream inStream2 = mFileSystem.openFile(file2);
Assert.assertNotNull(inStream2);
inStream2.close();
}
@Test
public void mountAfterMasterRestart() throws Exception {
mLocalAlluxioCluster.stopFS();
MasterRegistry registry = MasterTestUtils.createLeaderFileSystemMasterFromJournal();
FileSystemMaster fsMaster = registry.get(FileSystemMaster.class);
Map<String, MountPointInfo> mountTable = fsMaster.getMountTable();
Assert.assertTrue(mountTable.containsKey(MOUNT_POINT1));
Assert.assertTrue(mountTable.containsKey(MOUNT_POINT2));
MountPointInfo mountPointInfo1 = mountTable.get(MOUNT_POINT1);
MountPointInfo mountPointInfo2 = mountTable.get(MOUNT_POINT2);
Assert.assertEquals(mUfsUri1, mountPointInfo1.getUfsUri());
Assert.assertEquals(mUfsUri2, mountPointInfo2.getUfsUri());
Assert.assertEquals(UFS_CONF1, mountPointInfo1.getProperties());
Assert.assertEquals(UFS_CONF2, mountPointInfo2.getProperties());
}
@Test
public void mountAfterWorkerRestart() throws Exception {
String ufsFile1 = PathUtils.concatPath(mUfsUri1, "file1");
String ufsFile2 = PathUtils.concatPath(mUfsUri2, "file2");
UnderFileSystemUtils.touch(mLocalUfs, ufsFile1);
UnderFileSystemUtils.touch(mLocalUfs, ufsFile2);
mLocalAlluxioCluster.stopWorkers();
mLocalAlluxioCluster.startWorkers();
AlluxioURI file1 = mMountPoint1.join("file1");
AlluxioURI file2 = mMountPoint2.join("file2");
Assert.assertTrue(mFileSystem.exists(file1));
Assert.assertTrue(mFileSystem.exists(file2));
FileInStream inStream1 = mFileSystem.openFile(file1);
Assert.assertNotNull(inStream1);
inStream1.close();
FileInStream inStream2 = mFileSystem.openFile(file2);
Assert.assertNotNull(inStream2);
inStream2.close();
}
}