/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.tools.admin.command; import com.linkedin.pinot.common.data.FieldSpec; import com.linkedin.pinot.common.data.Schema; import com.linkedin.pinot.core.indexsegment.utils.AvroUtils; import com.linkedin.pinot.tools.Command; import org.codehaus.jackson.map.ObjectMapper; import org.kohsuke.args4j.Option; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.PrintWriter; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; /** * Class for command to convert avro schema to pinot schema. Given that it is not always possible to * automatically do this, the intention is to get most of the work done by this class, and require any * manual editing on top. */ public class AvroSchemaToPinotSchema extends AbstractBaseAdminCommand implements Command { private static final Logger LOGGER = LoggerFactory.getLogger(AddTenantCommand.class); @Option(name = "-avroSchemaFileName", required = false, forbids = {"-avroDataFileName"}, metaVar = "<String>", usage = "Path to avro schema file.") String _avroSchemaFileName; @Option(name = "-avroDataFileName", required = false, forbids = {"-avroSchemaFileName"}, metaVar = "<String>", usage = "Path to avro schema file.") String _avroDataFileName; @Option(name = "-pinotSchemaFileName", required = false, metaVar = "<string>", usage = "Path to pinot schema file.") String _pinotSchemaFileName; @Option(name = "-dimensions", required = false, metaVar = "<string>", usage = "Comma seperated dimension columns.") String _dimensions; @Option(name = "-metrics", required = false, metaVar = "<string>", usage = "Comma seperated metric columns.") String _metrics; @Option(name = "-timeColumnName", required = false, metaVar = "<string>", usage = "Name of Time Column.") String _timeColumnName; @Option(name = "-timeUnit", required = false, metaVar = "<string>", usage = "Unit for time column.") TimeUnit _timeUnit = TimeUnit.DAYS; @Option(name = "-help", required = false, help = true, aliases = { "-h", "--h", "--help" }, usage = "Print this message.") private boolean _help = false; @Override public boolean execute() throws Exception { Schema schema = null; if (_avroSchemaFileName != null) { schema = AvroUtils.getPinotSchemaFromAvroSchemaFile(_avroSchemaFileName, buildFieldTypesMap(), _timeUnit); } else if (_avroDataFileName != null) { schema = AvroUtils.extractSchemaFromAvro(new File(_avroDataFileName)); } else { LOGGER.error("Error: Missing required argument, please specify either -avroSchemaFileName, or -avroDataFileName"); return false; } if (schema == null) { LOGGER.error("Error: Could not read avro schema from file."); return false; } String avroFileName = _avroSchemaFileName; if (avroFileName == null) { avroFileName = _avroDataFileName; } // Remove extension String schemaName = avroFileName.replaceAll("\\..*", ""); schema.setSchemaName(schemaName); if (_pinotSchemaFileName == null) { _pinotSchemaFileName = schemaName + ".json"; LOGGER.info("Using {} as the Pinot schema file name", _pinotSchemaFileName); } ObjectMapper objectMapper = new ObjectMapper(); String schemaString = objectMapper.defaultPrettyPrintingWriter().writeValueAsString(schema); PrintWriter printWriter = new PrintWriter(_pinotSchemaFileName); printWriter.println(schemaString); printWriter.close(); return true; } @Override public String description() { return "Converting Avro schema to Pinot schema."; } @Override public boolean getHelp() { return _help; } @Override public String toString() { return "AvroSchemaToPinotSchema -avroSchemaFileName " + _avroSchemaFileName + " -pinotSchemaFileName " + _pinotSchemaFileName + " -dimensions " + _dimensions + " -metrics " + _metrics + " -timeColumnName " + _timeColumnName + " -timeUnit " + _timeUnit; } /** * Build a Map with column name as key and fieldType (dimension/metric/time) as value, from the * options list. * * @return The column <-> fieldType map. */ private Map<String, FieldSpec.FieldType> buildFieldTypesMap() { Map<String, FieldSpec.FieldType> fieldTypes = new HashMap<String, FieldSpec.FieldType>(); if (_dimensions != null) { for (String column : _dimensions.split("\\s*,\\s*")) { fieldTypes.put(column, FieldSpec.FieldType.DIMENSION); } } if (_metrics != null) { for (String column : _metrics.split("\\s*,\\s*")) { fieldTypes.put(column, FieldSpec.FieldType.METRIC); } } if (_timeColumnName != null) { fieldTypes.put(_timeColumnName, FieldSpec.FieldType.TIME); } return fieldTypes; } }