/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.hive.metastore;
import com.facebook.presto.hive.HiveType;
import com.facebook.presto.hive.PartitionNotFoundException;
import com.facebook.presto.hive.TableAlreadyExistsException;
import com.facebook.presto.spi.ColumnNotFoundException;
import com.facebook.presto.spi.PrestoException;
import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.TableNotFoundException;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.hive.metastore.TableType;
import javax.annotation.concurrent.GuardedBy;
import javax.annotation.concurrent.ThreadSafe;
import java.io.File;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR;
import static com.facebook.presto.hive.HiveUtil.toPartitionValues;
import static com.facebook.presto.hive.metastore.Database.DEFAULT_DATABASE_NAME;
import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP;
import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName;
import static com.facebook.presto.hive.metastore.PrincipalType.ROLE;
import static com.facebook.presto.hive.metastore.PrincipalType.USER;
import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS;
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
import static com.google.common.base.MoreObjects.toStringHelper;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.airlift.testing.FileUtils.deleteRecursively;
import static java.util.Collections.unmodifiableList;
import static java.util.Locale.US;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE;
import static org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE;
import static org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW;
@ThreadSafe
public class TestingHiveMetastore
implements ExtendedHiveMetastore
{
private static final String PUBLIC_ROLE_NAME = "public";
@GuardedBy("this")
private final Map<String, Database> databases = new HashMap<>();
@GuardedBy("this")
private final Map<SchemaTableName, Table> relations = new HashMap<>();
@GuardedBy("this")
private final Map<PartitionName, Partition> partitions = new HashMap<>();
@GuardedBy("this")
private final Map<String, Set<String>> roleGrants = new HashMap<>();
@GuardedBy("this")
private final Map<PrincipalTableKey, Set<HivePrivilegeInfo>> tablePrivileges = new HashMap<>();
private final File baseDirectory;
public TestingHiveMetastore(File baseDirectory)
{
this.baseDirectory = requireNonNull(baseDirectory, "baseDirectory is null");
checkArgument(!baseDirectory.exists(), "Base directory already exists");
checkArgument(baseDirectory.mkdirs(), "Could not create base directory");
}
@Override
public synchronized void createDatabase(Database database)
{
requireNonNull(database, "database is null");
if (!database.getLocation().isPresent()) {
File location = new File(baseDirectory, database.getDatabaseName() + ".db");
database = Database.builder(database)
.setLocation(Optional.of(location.getAbsoluteFile().toURI().toString()))
.build();
}
File directory = new File(URI.create(database.getLocation().get()));
if (directory.exists()) {
throw new PrestoException(HIVE_METASTORE_ERROR, "Database directory already exists");
}
if (!isParentDir(directory, baseDirectory)) {
throw new PrestoException(HIVE_METASTORE_ERROR, "Database directory must be inside of the metastore base directory");
}
if (!directory.mkdirs()) {
throw new PrestoException(HIVE_METASTORE_ERROR, "Could not create database directory");
}
if (databases.putIfAbsent(database.getDatabaseName(), database) != null) {
throw new PrestoException(ALREADY_EXISTS, "Database " + database.getDatabaseName() + " already exists");
}
}
@Override
public synchronized void dropDatabase(String databaseName)
{
if (relations.keySet().stream()
.map(SchemaTableName::getSchemaName)
.anyMatch(databaseName::equals)) {
throw new PrestoException(HIVE_METASTORE_ERROR, "Database " + databaseName + " is not empty");
}
databases.remove(databaseName);
}
@Override
public synchronized void renameDatabase(String databaseName, String newDatabaseName)
{
databases.remove(databaseName);
relations.values().forEach(table -> renameTable(table.getDatabaseName(), table.getTableName(), newDatabaseName, table.getTableName()));
// todo move data to new location
}
@Override
public synchronized List<String> getAllDatabases()
{
return ImmutableList.copyOf(databases.keySet());
}
@Override
public synchronized void createTable(Table table, PrincipalPrivileges principalPrivileges)
{
TableType tableType = TableType.valueOf(table.getTableType());
checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW).contains(tableType), "Invalid table type: %s", tableType);
SchemaTableName schemaTableName = new SchemaTableName(table.getDatabaseName(), table.getTableName());
if (tableType == VIRTUAL_VIEW) {
checkArgument(table.getStorage().getLocation().isEmpty(), "Storage location for view must be empty");
}
else {
File directory = new File(URI.create(table.getStorage().getLocation()));
if (!directory.exists()) {
throw new PrestoException(HIVE_METASTORE_ERROR, "Table directory does not exist");
}
if (tableType == MANAGED_TABLE && !isParentDir(directory, baseDirectory)) {
throw new PrestoException(HIVE_METASTORE_ERROR, "Table directory must be inside of the metastore base directory");
}
}
if (relations.putIfAbsent(schemaTableName, table) != null) {
throw new TableAlreadyExistsException(schemaTableName);
}
for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getUserPrivileges().asMap().entrySet()) {
setTablePrivileges(entry.getKey(), USER, table.getDatabaseName(), table.getTableName(), entry.getValue());
}
for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getRolePrivileges().asMap().entrySet()) {
setTablePrivileges(entry.getKey(), ROLE, table.getDatabaseName(), table.getTableName(), entry.getValue());
}
}
@Override
public synchronized void dropTable(String databaseName, String tableName, boolean deleteData)
{
SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
Table table = relations.remove(schemaTableName);
if (table == null) {
throw new TableNotFoundException(schemaTableName);
}
List<String> locations = listAllDataPaths(table);
// remove partitions
ImmutableList.copyOf(partitions.keySet()).stream()
.filter(partitionName -> partitionName.matches(databaseName, tableName))
.forEach(partitions::remove);
// remove permissions
ImmutableList.copyOf(tablePrivileges.keySet()).stream()
.filter(key -> key.matches(databaseName, tableName))
.forEach(tablePrivileges::remove);
// remove data
if (deleteData && table.getTableType().equals(MANAGED_TABLE.name())) {
for (String location : locations) {
File directory = new File(URI.create(location));
checkArgument(isParentDir(directory, baseDirectory), "Table directory must be inside of the metastore base directory");
deleteRecursively(directory);
}
}
}
private synchronized List<String> listAllDataPaths(Table table)
{
if (table.getTableType().equals(VIRTUAL_VIEW.name())) {
return ImmutableList.of();
}
ImmutableList.Builder<String> locations = ImmutableList.builder();
// For unpartitioned table, there should be nothing directly under this directory.
// But including this location in the set makes the directory content assert more
// extensive, which is desirable.
locations.add(table.getStorage().getLocation());
String tableStorageLocation = table.getStorage().getLocation().endsWith("/") ? table.getStorage().getLocation() : table.getStorage().getLocation() + "/";
partitions.entrySet().stream()
.filter(entry -> entry.getKey().matches(table.getDatabaseName(), table.getTableName()))
.map(entry -> entry.getValue().getStorage().getLocation())
.filter(location -> !location.startsWith(tableStorageLocation))
.forEach(locations::add);
return locations.build();
}
@Override
public synchronized void replaceTable(String databaseName, String tableName, Table newTable, PrincipalPrivileges principalPrivileges)
{
SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
Table table = getRequiredTable(schemaTableName);
checkArgument(newTable.getDatabaseName().equals(databaseName) && newTable.getTableName().equals(tableName), "Replacement table must have same name");
checkArgument(newTable.getTableType().equals(table.getTableType()), "Replacement table must have same type");
checkArgument(newTable.getStorage().getLocation().equals(table.getStorage().getLocation()), "Replacement table must have same location");
// replace table
relations.put(schemaTableName, newTable);
// remove old permissions
ImmutableList.copyOf(tablePrivileges.keySet()).stream()
.filter(key -> key.matches(databaseName, tableName))
.forEach(tablePrivileges::remove);
// add new permissions
for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getUserPrivileges().asMap().entrySet()) {
setTablePrivileges(entry.getKey(), USER, newTable.getDatabaseName(), newTable.getTableName(), entry.getValue());
}
for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getRolePrivileges().asMap().entrySet()) {
setTablePrivileges(entry.getKey(), ROLE, newTable.getDatabaseName(), newTable.getTableName(), entry.getValue());
}
}
@Override
public synchronized void renameTable(String databaseName, String tableName, String newDatabaseName, String newTableName)
{
SchemaTableName oldName = new SchemaTableName(databaseName, tableName);
Table oldTable = getRequiredTable(oldName);
// todo move data to new location
Table newTable = Table.builder(oldTable)
.setDatabaseName(newDatabaseName)
.setTableName(newTableName)
.build();
SchemaTableName newName = new SchemaTableName(newDatabaseName, newTableName);
if (relations.putIfAbsent(newName, newTable) != null) {
throw new TableAlreadyExistsException(newName);
}
relations.remove(oldName);
// rename partitions
for (Entry<PartitionName, Partition> entry : ImmutableList.copyOf(partitions.entrySet())) {
PartitionName partitionName = entry.getKey();
Partition partition = entry.getValue();
if (partitionName.matches(databaseName, tableName)) {
partitions.remove(partitionName);
partitions.put(
new PartitionName(newDatabaseName, newTableName, partitionName.getValues()),
Partition.builder(partition)
.setDatabaseName(newDatabaseName)
.setTableName(newTableName)
.build());
}
}
// rename privileges
for (Entry<PrincipalTableKey, Set<HivePrivilegeInfo>> entry : ImmutableList.copyOf(tablePrivileges.entrySet())) {
PrincipalTableKey principalTableKey = entry.getKey();
Set<HivePrivilegeInfo> privileges = entry.getValue();
if (principalTableKey.matches(databaseName, tableName)) {
tablePrivileges.remove(principalTableKey);
tablePrivileges.put(
new PrincipalTableKey(principalTableKey.getPrincipalName(), principalTableKey.getPrincipalType(), newTableName, newDatabaseName),
privileges);
}
}
}
@Override
public synchronized void addColumn(String databaseName, String tableName, String columnName, HiveType columnType, String columnComment)
{
SchemaTableName name = new SchemaTableName(databaseName, tableName);
Table oldTable = getRequiredTable(name);
if (oldTable.getColumn(columnName).isPresent()) {
throw new PrestoException(ALREADY_EXISTS, "Column already exists: " + columnName);
}
Table newTable = Table.builder(oldTable)
.addDataColumn(new Column(columnName, columnType, Optional.ofNullable(columnComment)))
.build();
relations.put(name, newTable);
}
@Override
public synchronized void renameColumn(String databaseName, String tableName, String oldColumnName, String newColumnName)
{
SchemaTableName name = new SchemaTableName(databaseName, tableName);
Table oldTable = getRequiredTable(name);
if (oldTable.getColumn(newColumnName).isPresent()) {
throw new PrestoException(ALREADY_EXISTS, "Column already exists: " + newColumnName);
}
if (!oldTable.getColumn(oldColumnName).isPresent()) {
throw new ColumnNotFoundException(name, oldColumnName);
}
for (Column column : oldTable.getPartitionColumns()) {
if (column.getName().equals(oldColumnName)) {
throw new PrestoException(NOT_SUPPORTED, "Renaming partition columns is not supported");
}
}
ImmutableList.Builder<Column> newDataColumns = ImmutableList.builder();
for (Column fieldSchema : oldTable.getDataColumns()) {
if (fieldSchema.getName().equals(oldColumnName)) {
newDataColumns.add(new Column(newColumnName, fieldSchema.getType(), fieldSchema.getComment()));
}
else {
newDataColumns.add(fieldSchema);
}
}
Table newTable = Table.builder(oldTable)
.setDataColumns(newDataColumns.build())
.build();
relations.put(name, newTable);
}
@Override
public synchronized Optional<List<String>> getAllTables(String databaseName)
{
if (!databases.containsKey(databaseName)) {
return Optional.empty();
}
return Optional.of(relations.keySet().stream()
.filter(name -> name.getSchemaName().equals(databaseName))
.map(SchemaTableName::getTableName)
.collect(toImmutableList()));
}
@Override
public synchronized Optional<List<String>> getAllViews(String databaseName)
{
if (!databases.containsKey(databaseName)) {
return Optional.empty();
}
List<String> views = relations.values().stream()
.filter(table -> table.getTableType().equals(VIRTUAL_VIEW.name()))
.map(Table::getTableName)
.collect(toImmutableList());
return Optional.of(views);
}
@Override
public synchronized Optional<Database> getDatabase(String databaseName)
{
return Optional.ofNullable(databases.get(databaseName));
}
@Override
public synchronized void addPartitions(String databaseName, String tableName, List<Partition> partitions)
{
for (Partition partition : partitions) {
PartitionName name = new PartitionName(databaseName, tableName, partition.getValues());
if (this.partitions.putIfAbsent(name, partition) != null) {
throw new PrestoException(HIVE_METASTORE_ERROR, "Partition already exists");
}
}
}
@Override
public synchronized void dropPartition(String databaseName, String tableName, List<String> parts, boolean deleteData)
{
Table table = getRequiredTable(new SchemaTableName(databaseName, tableName));
Partition partition = partitions.remove(new PartitionName(databaseName, tableName, parts));
if (partition == null) {
throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), parts);
}
if (deleteData && table.getTableType().equals(MANAGED_TABLE.name())) {
File directory = new File(URI.create(partition.getStorage().getLocation()));
checkArgument(isParentDir(directory, baseDirectory), "Partition directory must be inside of the metastore base directory");
deleteRecursively(directory);
}
}
@Override
public synchronized void alterPartition(String databaseName, String tableName, Partition partition)
{
PartitionName partitionName = new PartitionName(databaseName, tableName, partition.getValues());
Partition oldPartition = partitions.get(partitionName);
if (oldPartition == null) {
throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partition.getValues());
}
if (!oldPartition.getStorage().getLocation().equals(partition.getStorage().getLocation())) {
throw new PrestoException(HIVE_METASTORE_ERROR, "alterPartition can not change storage location");
}
dropPartition(databaseName, tableName, partition.getValues(), false);
addPartitions(databaseName, tableName, ImmutableList.of(partition));
}
@Override
public synchronized Optional<List<String>> getPartitionNames(String databaseName, String tableName)
{
return getTable(databaseName, tableName).map(table ->
partitions.entrySet().stream()
.filter(entry -> entry.getKey().matches(databaseName, tableName))
.map(entry -> entry.getKey().getPartitionName(table.getPartitionColumns()))
.collect(toImmutableList()));
}
@Override
public synchronized Optional<Partition> getPartition(String databaseName, String tableName, List<String> partitionValues)
{
PartitionName name = new PartitionName(databaseName, tableName, partitionValues);
return Optional.ofNullable(partitions.get(name));
}
@Override
public synchronized Optional<List<String>> getPartitionNamesByParts(String databaseName, String tableName, List<String> parts)
{
return getTable(databaseName, tableName).map(table ->
partitions.entrySet().stream()
.filter(entry -> partitionMatches(entry.getValue(), databaseName, tableName, parts))
.map(entry -> entry.getKey().getPartitionName(table.getPartitionColumns()))
.collect(toList()));
}
private static boolean partitionMatches(Partition partition, String databaseName, String tableName, List<String> parts)
{
if (!partition.getDatabaseName().equals(databaseName) ||
!partition.getTableName().equals(tableName)) {
return false;
}
List<String> values = partition.getValues();
if (values.size() != parts.size()) {
return false;
}
for (int i = 0; i < values.size(); i++) {
String part = parts.get(i);
if (!part.isEmpty() && !values.get(i).equals(part)) {
return false;
}
}
return true;
}
@Override
public synchronized Map<String, Optional<Partition>> getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames)
{
ImmutableMap.Builder<String, Optional<Partition>> builder = ImmutableMap.builder();
for (String name : partitionNames) {
PartitionName partitionName = new PartitionName(databaseName, tableName, toPartitionValues(name));
Partition partition = partitions.get(partitionName);
builder.put(name, Optional.ofNullable(partition));
}
return builder.build();
}
@Override
public synchronized Optional<Table> getTable(String databaseName, String tableName)
{
SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
return Optional.ofNullable(relations.get(schemaTableName));
}
private synchronized Table getRequiredTable(SchemaTableName tableName)
{
Table oldTable = relations.get(tableName);
if (oldTable == null) {
throw new TableNotFoundException(tableName);
}
return oldTable;
}
@Override
public synchronized Set<String> getRoles(String user)
{
return ImmutableSet.<String>builder()
.add(PUBLIC_ROLE_NAME)
.addAll(roleGrants.getOrDefault(user, ImmutableSet.of()))
.build();
}
public synchronized void setUserRoles(String user, ImmutableSet<String> roles)
{
roleGrants.put(user, roles);
}
@Override
public synchronized Set<HivePrivilegeInfo> getDatabasePrivileges(String user, String databaseName)
{
Set<HivePrivilegeInfo> privileges = new HashSet<>();
if (isDatabaseOwner(user, databaseName)) {
privileges.add(new HivePrivilegeInfo(OWNERSHIP, true));
}
return privileges;
}
@Override
public synchronized Set<HivePrivilegeInfo> getTablePrivileges(String user, String databaseName, String tableName)
{
Set<HivePrivilegeInfo> privileges = new HashSet<>();
if (isTableOwner(user, databaseName, tableName)) {
privileges.add(new HivePrivilegeInfo(OWNERSHIP, true));
}
privileges.addAll(tablePrivileges.getOrDefault(new PrincipalTableKey(user, USER, tableName, databaseName), ImmutableSet.of()));
for (String role : getRoles(user)) {
privileges.addAll(tablePrivileges.getOrDefault(new PrincipalTableKey(role, ROLE, tableName, databaseName), ImmutableSet.of()));
}
return privileges;
}
private synchronized void setTablePrivileges(String principalName,
PrincipalType principalType,
String databaseName,
String tableName,
Iterable<HivePrivilegeInfo> privileges)
{
tablePrivileges.put(new PrincipalTableKey(principalName, principalType, tableName, databaseName), ImmutableSet.copyOf(privileges));
}
@Override
public synchronized void grantTablePrivileges(String databaseName, String tableName, String grantee, Set<HivePrivilegeInfo> privileges)
{
setTablePrivileges(grantee, USER, databaseName, tableName, privileges);
}
@Override
public synchronized void revokeTablePrivileges(String databaseName, String tableName, String grantee, Set<HivePrivilegeInfo> privileges)
{
Set<HivePrivilegeInfo> currentPrivileges = getTablePrivileges(grantee, databaseName, tableName);
currentPrivileges.removeAll(privileges);
setTablePrivileges(grantee, USER, databaseName, tableName, currentPrivileges);
}
private static boolean isParentDir(File directory, File baseDirectory)
{
return directory.toPath().startsWith(baseDirectory.toPath());
}
private boolean isDatabaseOwner(String user, String databaseName)
{
// all users are "owners" of the default database
if (DEFAULT_DATABASE_NAME.equalsIgnoreCase(databaseName)) {
return true;
}
Optional<Database> databaseMetadata = getDatabase(databaseName);
if (!databaseMetadata.isPresent()) {
return false;
}
Database database = databaseMetadata.get();
// a database can be owned by a user or role
if (database.getOwnerType() == USER && user.equals(database.getOwnerName())) {
return true;
}
if (database.getOwnerType() == ROLE && getRoles(user).contains(database.getOwnerName())) {
return true;
}
return false;
}
private boolean isTableOwner(String user, String databaseName, String tableName)
{
// a table can only be owned by a user
Optional<Table> table = getTable(databaseName, tableName);
return table.isPresent() && user.equals(table.get().getOwner());
}
private static class PartitionName
{
private final String schemaName;
private final String tableName;
private final List<String> values;
public PartitionName(String schemaName, String tableName, List<String> values)
{
this.schemaName = schemaName.toLowerCase(US);
this.tableName = tableName.toLowerCase(US);
this.values = unmodifiableList(new ArrayList<>(values));
}
public List<String> getValues()
{
return values;
}
public String getPartitionName(List<Column> partitionColumns)
{
return makePartName(partitionColumns, values);
}
public boolean matches(String schemaName, String tableName)
{
return this.schemaName.equals(schemaName) &&
this.tableName.equals(tableName);
}
@Override
public int hashCode()
{
return Objects.hash(schemaName, tableName, values);
}
@Override
public boolean equals(Object obj)
{
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
PartitionName other = (PartitionName) obj;
return Objects.equals(this.schemaName, other.schemaName)
&& Objects.equals(this.tableName, other.tableName)
&& Objects.equals(this.values, other.values);
}
@Override
public String toString()
{
return schemaName + "/" + tableName + "/" + values;
}
}
private static class PrincipalTableKey
{
private final String principalName;
private final PrincipalType principalType;
private final String database;
private final String table;
public PrincipalTableKey(String principalName, PrincipalType principalType, String table, String database)
{
this.principalName = requireNonNull(principalName, "principalName is null");
this.principalType = requireNonNull(principalType, "principalType is null");
this.table = requireNonNull(table, "table is null");
this.database = requireNonNull(database, "database is null");
}
public String getPrincipalName()
{
return principalName;
}
public PrincipalType getPrincipalType()
{
return principalType;
}
public boolean matches(String databaseName, String tableName)
{
return this.database.equals(databaseName) && this.table.equals(tableName);
}
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
PrincipalTableKey that = (PrincipalTableKey) o;
return Objects.equals(principalName, that.principalName) &&
Objects.equals(principalType, that.principalType) &&
Objects.equals(table, that.table) &&
Objects.equals(database, that.database);
}
@Override
public int hashCode()
{
return Objects.hash(principalName, principalType, table, database);
}
@Override
public String toString()
{
return toStringHelper(this)
.add("principalName", principalName)
.add("principalType", principalType)
.add("table", table)
.add("database", database)
.toString();
}
}
}