/* * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 * (the "License"). You may not use this work except in compliance with the License, which is * available at www.apache.org/licenses/LICENSE-2.0 * * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, * either express or implied, as more fully set forth in the License. * * See the NOTICE file distributed with this work for information regarding copyright ownership. */ package alluxio.underfs; import alluxio.AlluxioURI; import alluxio.Configuration; import alluxio.PropertyKey; import alluxio.util.IdUtils; import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.io.Closer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** * Basic implementation of {@link UfsManager}. */ public abstract class AbstractUfsManager implements UfsManager { private static final Logger LOG = LoggerFactory.getLogger(AbstractUfsManager.class); /** * The key of the UFS cache. */ public static class Key { private final String mScheme; private final String mAuthority; private final Map<String, String> mProperties; Key(AlluxioURI uri, Map<String, String> properties) { mScheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(); mAuthority = uri.getAuthority() == null ? "" : uri.getAuthority().toLowerCase(); mProperties = (properties == null || properties.isEmpty()) ? null : properties; } @Override public int hashCode() { return Objects.hashCode(mScheme, mAuthority, mProperties); } @Override public boolean equals(Object object) { if (object == this) { return true; } if (!(object instanceof Key)) { return false; } Key that = (Key) object; return Objects.equal(mAuthority, that.mAuthority) && Objects .equal(mProperties, that.mProperties) && Objects.equal(mScheme, that.mScheme); } @Override public String toString() { return Objects.toStringHelper(this) .add("authority", mAuthority) .add("scheme", mScheme) .add("properties", mProperties) .toString(); } } // TODO(binfan): Add refcount to the UFS instance. Once the refcount goes to zero, // we could close this UFS instance. /** * Maps from key to {@link UnderFileSystem} instances. This map keeps the entire set of UFS * instances, each keyed by their unique combination of Uri and conf information. This map * helps efficiently identify if a UFS instance in request should be created or can be reused. */ private final ConcurrentHashMap<Key, UnderFileSystem> mUnderFileSystemMap = new ConcurrentHashMap<>(); /** * Maps from mount id to {@link UnderFileSystem} instances. This map helps efficiently retrieve * an existing UFS instance given its mount id. */ private final ConcurrentHashMap<Long, UnderFileSystem> mMountIdToUnderFileSystemMap = new ConcurrentHashMap<>(); private UnderFileSystem mRootUfs; protected final Closer mCloser; AbstractUfsManager() { mCloser = Closer.create(); } /** * Establishes the connection to the given UFS from the server. * * @param ufs UFS instance * @throws IOException if failed to create the UFS instance */ protected abstract void connect(UnderFileSystem ufs) throws IOException; /** * Return a UFS instance if it already exists in the cache, otherwise, creates a new instance and * return this. * * @param ufsUri the UFS path * @param ufsConf the UFS configuration * @return the UFS instance * @throws IOException if it is failed to create the UFS instance */ private UnderFileSystem getOrAdd(String ufsUri, UnderFileSystemConfiguration ufsConf) throws IOException { Key key = new Key(new AlluxioURI(ufsUri), ufsConf.getUserSpecifiedConf()); UnderFileSystem cachedFs = mUnderFileSystemMap.get(key); if (cachedFs != null) { return cachedFs; } UnderFileSystem fs = UnderFileSystem.Factory.create(ufsUri, ufsConf); try { connect(fs); } catch (IOException e) { fs.close(); throw e; } cachedFs = mUnderFileSystemMap.putIfAbsent(key, fs); if (cachedFs == null) { // above insert is successful mCloser.register(fs); return fs; } try { fs.close(); } catch (IOException e) { // Cannot close the created ufs which fails the race. LOG.error("Failed to close UFS {}", fs, e); } return cachedFs; } @Override public UnderFileSystem addMount(long mountId, String ufsUri, UnderFileSystemConfiguration ufsConf) throws IOException { Preconditions.checkArgument(mountId != IdUtils.INVALID_MOUNT_ID, "mountId"); Preconditions.checkArgument(ufsUri != null, "uri"); Preconditions.checkArgument(ufsConf != null, "ufsConf"); UnderFileSystem ufs = getOrAdd(ufsUri, ufsConf); mMountIdToUnderFileSystemMap.put(mountId, ufs); return ufs; } @Override public void removeMount(long mountId) { Preconditions.checkArgument(mountId != IdUtils.INVALID_MOUNT_ID, "mountId"); // TODO(binfan): check the refcount of this ufs in mUnderFileSystemMap and remove it if this is // no more used. Currently, it is possibly used by out mount too. mMountIdToUnderFileSystemMap.remove(mountId); } @Override public UnderFileSystem get(long mountId) { return mMountIdToUnderFileSystemMap.get(mountId); } @Override public UnderFileSystem getRoot() { synchronized (this) { if (mRootUfs == null) { String rootUri = Configuration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); boolean rootReadOnly = Configuration.getBoolean(PropertyKey.MASTER_MOUNT_TABLE_ROOT_READONLY); boolean rootShared = Configuration.getBoolean(PropertyKey.MASTER_MOUNT_TABLE_ROOT_SHARED); Map<String, String> rootConf = Configuration.getNestedProperties(PropertyKey.MASTER_MOUNT_TABLE_ROOT_OPTION); try { mRootUfs = addMount(IdUtils.ROOT_MOUNT_ID, rootUri, new UnderFileSystemConfiguration(rootReadOnly, rootShared, rootConf)); } catch (IOException e) { throw new RuntimeException(e); } } return mRootUfs; } } @Override public void close() throws IOException { mCloser.close(); } }