/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.hive.metastore;
import com.facebook.presto.hive.HiveBucketProperty;
import com.facebook.presto.hive.HiveType;
import com.facebook.presto.spi.PrestoException;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.metastore.ProtectMode;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA;
import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.parsePrivilege;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Strings.emptyToNull;
import static com.google.common.base.Strings.nullToEmpty;
import static java.lang.String.format;
import static java.util.stream.Collectors.toList;
import static org.apache.hadoop.hive.metastore.MetaStoreUtils.typeToThriftType;
import static org.apache.hadoop.hive.metastore.ProtectMode.getProtectModeFromString;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_COUNT;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_FIELD_NAME;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_OUTPUT_FORMAT;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES;
import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_DDL;
import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB;
public class MetastoreUtil
{
private MetastoreUtil()
{
}
public static Properties getHiveSchema(Table table)
{
// Mimics function in Hive: MetaStoreUtils.getTableMetadata(Table)
return getHiveSchema(
table.getStorage(),
table.getDataColumns(),
table.getDataColumns(),
table.getParameters(),
table.getDatabaseName(),
table.getTableName(),
table.getPartitionColumns());
}
public static Properties getHiveSchema(Partition partition, Table table)
{
// Mimics function in Hive: MetaStoreUtils.getSchema(Partition, Table)
return getHiveSchema(
partition.getStorage(),
partition.getColumns(),
table.getDataColumns(),
table.getParameters(),
table.getDatabaseName(),
table.getTableName(),
table.getPartitionColumns());
}
private static Properties getHiveSchema(
Storage sd,
List<Column> dataColumns,
List<Column> tableDataColumns,
Map<String, String> parameters,
String databaseName,
String tableName,
List<Column> partitionKeys)
{
// Mimics function in Hive:
// MetaStoreUtils.getSchema(StorageDescriptor, StorageDescriptor, Map<String, String>, String, String, List<FieldSchema>)
Properties schema = new Properties();
schema.setProperty(FILE_INPUT_FORMAT, sd.getStorageFormat().getInputFormat());
schema.setProperty(FILE_OUTPUT_FORMAT, sd.getStorageFormat().getOutputFormat());
schema.setProperty(META_TABLE_NAME, databaseName + "." + tableName);
schema.setProperty(META_TABLE_LOCATION, sd.getLocation());
if (sd.getBucketProperty().isPresent()) {
schema.setProperty(BUCKET_FIELD_NAME, sd.getBucketProperty().get().getBucketedBy().get(0));
schema.setProperty(BUCKET_COUNT, Integer.toString(sd.getBucketProperty().get().getBucketCount()));
}
else {
schema.setProperty(BUCKET_COUNT, "0");
}
for (Map.Entry<String, String> param : sd.getSerdeParameters().entrySet()) {
schema.setProperty(param.getKey(), (param.getValue() != null) ? param.getValue() : "");
}
schema.setProperty(SERIALIZATION_LIB, sd.getStorageFormat().getSerDe());
StringBuilder columnNameBuilder = new StringBuilder();
StringBuilder columnTypeBuilder = new StringBuilder();
StringBuilder columnCommentBuilder = new StringBuilder();
boolean first = true;
for (Column column : tableDataColumns) {
if (!first) {
columnNameBuilder.append(",");
columnTypeBuilder.append(":");
columnCommentBuilder.append('\0');
}
columnNameBuilder.append(column.getName());
columnTypeBuilder.append(column.getType());
columnCommentBuilder.append(column.getComment().orElse(""));
first = false;
}
String columnNames = columnNameBuilder.toString();
String columnTypes = columnTypeBuilder.toString();
schema.setProperty(META_TABLE_COLUMNS, columnNames);
schema.setProperty(META_TABLE_COLUMN_TYPES, columnTypes);
schema.setProperty("columns.comments", columnCommentBuilder.toString());
schema.setProperty(SERIALIZATION_DDL, toThriftDdl(tableName, dataColumns));
String partString = "";
String partStringSep = "";
String partTypesString = "";
String partTypesStringSep = "";
for (Column partKey : partitionKeys) {
partString += partStringSep;
partString += partKey.getName();
partTypesString += partTypesStringSep;
partTypesString += partKey.getType().getHiveTypeName();
if (partStringSep.length() == 0) {
partStringSep = "/";
partTypesStringSep = ":";
}
}
if (partString.length() > 0) {
schema.setProperty(META_TABLE_PARTITION_COLUMNS, partString);
schema.setProperty(META_TABLE_PARTITION_COLUMN_TYPES, partTypesString);
}
if (parameters != null) {
for (Map.Entry<String, String> entry : parameters.entrySet()) {
// add non-null parameters to the schema
if (entry.getValue() != null) {
schema.setProperty(entry.getKey(), entry.getValue());
}
}
}
return schema;
}
public static ProtectMode getProtectMode(Partition partition)
{
return getProtectMode(partition.getParameters());
}
public static ProtectMode getProtectMode(Table table)
{
return getProtectMode(table.getParameters());
}
public static String makePartName(List<Column> partitionColumns, List<String> values)
{
checkArgument(partitionColumns.size() == values.size());
List<String> partitionColumnNames = partitionColumns.stream().map(Column::getName).collect(toList());
return FileUtils.makePartName(partitionColumnNames, values);
}
public static org.apache.hadoop.hive.metastore.api.Database toMetastoreApiDatabase(Database database)
{
org.apache.hadoop.hive.metastore.api.Database result = new org.apache.hadoop.hive.metastore.api.Database();
result.setName(database.getDatabaseName());
database.getLocation().ifPresent(result::setLocationUri);
result.setOwnerName(database.getOwnerName());
result.setOwnerType(toMetastoreApiPrincipalType(database.getOwnerType()));
database.getComment().ifPresent(result::setDescription);
result.setParameters(database.getParameters());
return result;
}
public static org.apache.hadoop.hive.metastore.api.Table toMetastoreApiTable(Table table, PrincipalPrivileges privileges)
{
org.apache.hadoop.hive.metastore.api.Table result = new org.apache.hadoop.hive.metastore.api.Table();
result.setDbName(table.getDatabaseName());
result.setTableName(table.getTableName());
result.setOwner(table.getOwner());
result.setTableType(table.getTableType());
result.setParameters(table.getParameters());
result.setPartitionKeys(table.getPartitionColumns().stream().map(MetastoreUtil::toMetastoreApiFieldSchema).collect(toList()));
result.setSd(makeStorageDescriptor(table.getTableName(), table.getDataColumns(), table.getStorage()));
result.setPrivileges(toMetastoreApiPrincipalPrivilegeSet(table.getOwner(), privileges));
result.setViewOriginalText(table.getViewOriginalText().orElse(null));
result.setViewExpandedText(table.getViewExpandedText().orElse(null));
return result;
}
private static PrincipalPrivilegeSet toMetastoreApiPrincipalPrivilegeSet(String grantee, PrincipalPrivileges privileges)
{
ImmutableMap.Builder<String, List<PrivilegeGrantInfo>> userPrivileges = ImmutableMap.builder();
for (Entry<String, Collection<HivePrivilegeInfo>> entry : privileges.getUserPrivileges().asMap().entrySet()) {
userPrivileges.put(entry.getKey(), entry.getValue().stream()
.map(privilegeInfo -> toMetastoreApiPrivilegeGrantInfo(grantee, privilegeInfo))
.collect(toList()));
}
ImmutableMap.Builder<String, List<PrivilegeGrantInfo>> rolePrivileges = ImmutableMap.builder();
for (Entry<String, Collection<HivePrivilegeInfo>> entry : privileges.getRolePrivileges().asMap().entrySet()) {
rolePrivileges.put(entry.getKey(), entry.getValue().stream()
.map(privilegeInfo -> toMetastoreApiPrivilegeGrantInfo(grantee, privilegeInfo))
.collect(toList()));
}
return new PrincipalPrivilegeSet(userPrivileges.build(), ImmutableMap.of(), rolePrivileges.build());
}
public static PrivilegeGrantInfo toMetastoreApiPrivilegeGrantInfo(String grantee, HivePrivilegeInfo privilegeInfo)
{
return new PrivilegeGrantInfo(
privilegeInfo.getHivePrivilege().name().toLowerCase(),
0,
grantee,
org.apache.hadoop.hive.metastore.api.PrincipalType.USER, privilegeInfo.isGrantOption());
}
private static org.apache.hadoop.hive.metastore.api.PrincipalType toMetastoreApiPrincipalType(PrincipalType principalType)
{
switch (principalType) {
case USER:
return org.apache.hadoop.hive.metastore.api.PrincipalType.USER;
case ROLE:
return org.apache.hadoop.hive.metastore.api.PrincipalType.ROLE;
default:
throw new IllegalArgumentException("Unsupported principal type: " + principalType);
}
}
public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(Partition partition)
{
org.apache.hadoop.hive.metastore.api.Partition result = new org.apache.hadoop.hive.metastore.api.Partition();
result.setDbName(partition.getDatabaseName());
result.setTableName(partition.getTableName());
result.setValues(partition.getValues());
result.setSd(makeStorageDescriptor(partition.getTableName(), partition.getColumns(), partition.getStorage()));
result.setParameters(partition.getParameters());
return result;
}
public static Database fromMetastoreApiDatabase(org.apache.hadoop.hive.metastore.api.Database database)
{
String ownerName = "PUBLIC";
PrincipalType ownerType = PrincipalType.ROLE;
if (database.getOwnerName() != null) {
ownerName = database.getOwnerName();
ownerType = fromMetastoreApiPrincipalType(database.getOwnerType());
}
Map<String, String> parameters = database.getParameters();
if (parameters == null) {
parameters = ImmutableMap.of();
}
return Database.builder()
.setDatabaseName(database.getName())
.setLocation(Optional.ofNullable(database.getLocationUri()))
.setOwnerName(ownerName)
.setOwnerType(ownerType)
.setComment(Optional.ofNullable(database.getDescription()))
.setParameters(parameters)
.build();
}
public static Table fromMetastoreApiTable(org.apache.hadoop.hive.metastore.api.Table table)
{
StorageDescriptor storageDescriptor = table.getSd();
if (storageDescriptor == null) {
throw new PrestoException(HIVE_INVALID_METADATA, "Table is missing storage descriptor");
}
Table.Builder tableBuilder = Table.builder()
.setDatabaseName(table.getDbName())
.setTableName(table.getTableName())
.setOwner(nullToEmpty(table.getOwner()))
.setTableType(table.getTableType())
.setDataColumns(storageDescriptor.getCols().stream()
.map(MetastoreUtil::fromMetastoreApiFieldSchema)
.collect(toList()))
.setPartitionColumns(table.getPartitionKeys().stream()
.map(MetastoreUtil::fromMetastoreApiFieldSchema)
.collect(toList()))
.setParameters(table.getParameters() == null ? ImmutableMap.of() : table.getParameters())
.setViewOriginalText(Optional.ofNullable(emptyToNull(table.getViewOriginalText())))
.setViewExpandedText(Optional.ofNullable(emptyToNull(table.getViewExpandedText())));
fromMetastoreApiStorageDescriptor(storageDescriptor, tableBuilder.getStorageBuilder(), table.getTableName());
return tableBuilder.build();
}
public static Partition fromMetastoreApiPartition(org.apache.hadoop.hive.metastore.api.Partition partition)
{
StorageDescriptor storageDescriptor = partition.getSd();
if (storageDescriptor == null) {
throw new PrestoException(HIVE_INVALID_METADATA, "Partition does not contain a storage descriptor: " + partition);
}
Partition.Builder partitionBuilder = Partition.builder()
.setDatabaseName(partition.getDbName())
.setTableName(partition.getTableName())
.setValues(partition.getValues())
.setColumns(storageDescriptor.getCols().stream()
.map(MetastoreUtil::fromMetastoreApiFieldSchema)
.collect(toList()))
.setParameters(partition.getParameters());
fromMetastoreApiStorageDescriptor(storageDescriptor, partitionBuilder.getStorageBuilder(), format("%s.%s", partition.getTableName(), partition.getValues()));
return partitionBuilder.build();
}
private static PrincipalType fromMetastoreApiPrincipalType(org.apache.hadoop.hive.metastore.api.PrincipalType principalType)
{
switch (principalType) {
case USER:
return PrincipalType.USER;
case ROLE:
return PrincipalType.ROLE;
default:
throw new IllegalArgumentException("Unsupported principal type: " + principalType);
}
}
public static Set<HivePrivilegeInfo> toGrants(List<PrivilegeGrantInfo> userGrants)
{
if (userGrants == null) {
return ImmutableSet.of();
}
ImmutableSet.Builder<HivePrivilegeInfo> privileges = ImmutableSet.builder();
for (PrivilegeGrantInfo userGrant : userGrants) {
privileges.addAll(parsePrivilege(userGrant));
}
return privileges.build();
}
private static String toThriftDdl(String structName, List<Column> columns)
{
// Mimics function in Hive:
// MetaStoreUtils.getDDLFromFieldSchema(String, List<FieldSchema>)
StringBuilder ddl = new StringBuilder();
ddl.append("struct ");
ddl.append(structName);
ddl.append(" { ");
boolean first = true;
for (Column column : columns) {
if (first) {
first = false;
}
else {
ddl.append(", ");
}
ddl.append(typeToThriftType(column.getType().getHiveTypeName()));
ddl.append(' ');
ddl.append(column.getName());
}
ddl.append("}");
return ddl.toString();
}
private static ProtectMode getProtectMode(Map<String, String> parameters)
{
if (!parameters.containsKey(ProtectMode.PARAMETER_NAME)) {
return new ProtectMode();
}
else {
return getProtectModeFromString(parameters.get(ProtectMode.PARAMETER_NAME));
}
}
private static StorageDescriptor makeStorageDescriptor(String tableName, List<Column> columns, Storage storage)
{
if (storage.isSorted() || storage.isSkewed()) {
throw new IllegalArgumentException("Writing to sorted and/or skewed table/partition is not supported");
}
SerDeInfo serdeInfo = new SerDeInfo();
serdeInfo.setName(tableName);
serdeInfo.setSerializationLib(storage.getStorageFormat().getSerDeNullable());
serdeInfo.setParameters(storage.getSerdeParameters());
StorageDescriptor sd = new StorageDescriptor();
sd.setLocation(emptyToNull(storage.getLocation()));
sd.setCols(columns.stream()
.map(MetastoreUtil::toMetastoreApiFieldSchema)
.collect(toList()));
sd.setSerdeInfo(serdeInfo);
sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable());
sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable());
sd.setParameters(ImmutableMap.of());
Optional<HiveBucketProperty> bucketProperty = storage.getBucketProperty();
if (bucketProperty.isPresent()) {
sd.setNumBuckets(bucketProperty.get().getBucketCount());
sd.setBucketCols(bucketProperty.get().getBucketedBy());
}
return sd;
}
private static FieldSchema toMetastoreApiFieldSchema(Column column)
{
return new FieldSchema(column.getName(), column.getType().getHiveTypeName(), column.getComment().orElse(null));
}
private static Column fromMetastoreApiFieldSchema(FieldSchema fieldSchema)
{
return new Column(fieldSchema.getName(), HiveType.valueOf(fieldSchema.getType()), Optional.ofNullable(emptyToNull(fieldSchema.getComment())));
}
private static void fromMetastoreApiStorageDescriptor(StorageDescriptor storageDescriptor, Storage.Builder builder, String tablePartitionName)
{
SerDeInfo serdeInfo = storageDescriptor.getSerdeInfo();
if (serdeInfo == null) {
throw new PrestoException(HIVE_INVALID_METADATA, "Table storage descriptor is missing SerDe info");
}
builder.setStorageFormat(StorageFormat.createNullable(serdeInfo.getSerializationLib(), storageDescriptor.getInputFormat(), storageDescriptor.getOutputFormat()))
.setLocation(nullToEmpty(storageDescriptor.getLocation()))
.setBucketProperty(HiveBucketProperty.fromStorageDescriptor(storageDescriptor, tablePartitionName))
.setSorted(storageDescriptor.isSetSortCols() && !storageDescriptor.getSortCols().isEmpty())
.setSkewed(storageDescriptor.isSetSkewedInfo() && storageDescriptor.getSkewedInfo().isSetSkewedColNames() && !storageDescriptor.getSkewedInfo().getSkewedColNames().isEmpty())
.setSerdeParameters(serdeInfo.getParameters() == null ? ImmutableMap.of() : serdeInfo.getParameters());
}
}