/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.kafka;
import com.facebook.presto.decoder.dummy.DummyRowDecoder;
import com.facebook.presto.spi.SchemaTableName;
import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.airlift.json.JsonCodec;
import io.airlift.log.Logger;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Strings.isNullOrEmpty;
import static java.nio.file.Files.readAllBytes;
import static java.util.Arrays.asList;
import static java.util.Objects.requireNonNull;
public class KafkaTableDescriptionSupplier
implements Supplier<Map<SchemaTableName, KafkaTopicDescription>>
{
private static final Logger log = Logger.get(KafkaTableDescriptionSupplier.class);
private final JsonCodec<KafkaTopicDescription> topicDescriptionCodec;
private final File tableDescriptionDir;
private final String defaultSchema;
private final Set<String> tableNames;
@Inject
KafkaTableDescriptionSupplier(KafkaConnectorConfig kafkaConnectorConfig,
JsonCodec<KafkaTopicDescription> topicDescriptionCodec)
{
this.topicDescriptionCodec = requireNonNull(topicDescriptionCodec, "topicDescriptionCodec is null");
requireNonNull(kafkaConnectorConfig, "kafkaConfig is null");
this.tableDescriptionDir = kafkaConnectorConfig.getTableDescriptionDir();
this.defaultSchema = kafkaConnectorConfig.getDefaultSchema();
this.tableNames = ImmutableSet.copyOf(kafkaConnectorConfig.getTableNames());
}
@Override
public Map<SchemaTableName, KafkaTopicDescription> get()
{
ImmutableMap.Builder<SchemaTableName, KafkaTopicDescription> builder = ImmutableMap.builder();
log.debug("Loading kafka table definitions from %s", tableDescriptionDir.getAbsolutePath());
try {
for (File file : listFiles(tableDescriptionDir)) {
if (file.isFile() && file.getName().endsWith(".json")) {
KafkaTopicDescription table = topicDescriptionCodec.fromJson(readAllBytes(file.toPath()));
String schemaName = firstNonNull(table.getSchemaName(), defaultSchema);
log.debug("Kafka table %s.%s: %s", schemaName, table.getTableName(), table);
builder.put(new SchemaTableName(schemaName, table.getTableName()), table);
}
}
Map<SchemaTableName, KafkaTopicDescription> tableDefinitions = builder.build();
log.debug("Loaded Table definitions: %s", tableDefinitions.keySet());
builder = ImmutableMap.builder();
for (String definedTable : tableNames) {
SchemaTableName tableName;
try {
tableName = parseTableName(definedTable);
}
catch (IllegalArgumentException iae) {
tableName = new SchemaTableName(defaultSchema, definedTable);
}
if (tableDefinitions.containsKey(tableName)) {
KafkaTopicDescription kafkaTable = tableDefinitions.get(tableName);
log.debug("Found Table definition for %s: %s", tableName, kafkaTable);
builder.put(tableName, kafkaTable);
}
else {
// A dummy table definition only supports the internal columns.
log.debug("Created dummy Table definition for %s", tableName);
builder.put(tableName, new KafkaTopicDescription(tableName.getTableName(),
tableName.getSchemaName(),
definedTable,
new KafkaTopicFieldGroup(DummyRowDecoder.NAME, ImmutableList.of()),
new KafkaTopicFieldGroup(DummyRowDecoder.NAME, ImmutableList.of())));
}
}
return builder.build();
}
catch (IOException e) {
log.warn(e, "Error: ");
throw Throwables.propagate(e);
}
}
private static List<File> listFiles(File dir)
{
if ((dir != null) && dir.isDirectory()) {
File[] files = dir.listFiles();
if (files != null) {
log.debug("Considering files: %s", asList(files));
return ImmutableList.copyOf(files);
}
}
return ImmutableList.of();
}
private static SchemaTableName parseTableName(String schemaTableName)
{
checkArgument(!isNullOrEmpty(schemaTableName), "schemaTableName is null or is empty");
List<String> parts = Splitter.on('.').splitToList(schemaTableName);
checkArgument(parts.size() == 2, "Invalid schemaTableName: %s", schemaTableName);
return new SchemaTableName(parts.get(0), parts.get(1));
}
}