package com.linkedin.databus.bootstrap.utils;
/*
*
* Copyright 2013 LinkedIn Corp. All rights reserved
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.PropertyConfigurator;
import com.linkedin.databus.bootstrap.common.BootstrapConn;
import com.linkedin.databus.client.DbusEventAvroDecoder;
import com.linkedin.databus.core.DbusEventFactory;
import com.linkedin.databus.core.DbusEventInternalReadable;
import com.linkedin.databus.core.DbusEventV1Factory;
import com.linkedin.databus.core.util.ConfigBuilder;
import com.linkedin.databus.core.util.ConfigLoader;
import com.linkedin.databus.core.util.InvalidConfigException;
import com.linkedin.databus2.schemas.FileSystemSchemaRegistryService;
import com.linkedin.databus2.schemas.SchemaRegistryService;
import com.linkedin.databus2.schemas.VersionedSchema;
import com.linkedin.databus2.schemas.VersionedSchemaSet;
import com.linkedin.databus2.util.DBHelper;
public class BootstrapTableReader
{
public static final String MODULE = BootstrapDBReader.class.getName();
public static final Logger LOG = Logger.getLogger(MODULE);
public static final String HELP_OPT_LONG_NAME = "help";
public static final String BOOTSTRAP_DB_PROPS_OPT_LONG_NAME = "bootstrap_db_props";
public static final String QUERY_CONFIG_OPT_LONG_NAME = "query_config";
public static final String SOURCE_ID_PARAM = "source_id";
public static final String LOG4J_PROPS_OPT_LONG_NAME = "log_props";
public static final char HELP_OPT_CHAR = 'h';
public static final char BOOTSTRAP_DB_PROP_OPT_CHAR = 'p';
public static final char QUERY_CONFIG_OPT_CHAR = 'q';
public static final char LOG4J_PROPS_OPT_CHAR = 'l';
private static Properties _sBootstrapConfigProps = null;
private static Properties _sQueryConfigProps = null;
private static BootstrapSeederMain.StaticConfig _bsStaticConfig;
private static StaticConfig _queryStaticConfig;
private static DbusEventAvroDecoder _decoder;
private static Schema _schema;
private final DbusEventFactory _eventFactory;
private BootstrapReaderEventHandler _eventHandler;
private BootstrapConn _bootstrapConn = null;
public BootstrapTableReader(BootstrapReaderEventHandler eventHandler)
{
_eventFactory = new DbusEventV1Factory();
_eventHandler = eventHandler;
}
private String getTableName()
{
String table = null;
if (_queryStaticConfig.getTablePrefix().equalsIgnoreCase("tab_"))
{
table = "tab_" + _queryStaticConfig.getSourceId();
}
else
{
table = "log_" + _queryStaticConfig.getSourceId() + "_" + _queryStaticConfig.getLogId();
}
return table;
}
public void execute() throws Exception
{
String query = getQuery();
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try
{
conn = getConnection();
stmt = conn.createStatement();
LOG.info("Executing query : " + query);
rs = stmt.executeQuery(query);
byte[] b1 = new byte[1024 * 1024];
ByteBuffer buffer = ByteBuffer.wrap(b1);
DbusEventInternalReadable event = _eventFactory.createReadOnlyDbusEventFromBuffer(buffer, 0);
int count = 0;
_eventHandler.onStart(query);
while (rs.next())
{
buffer.clear();
buffer.put(rs.getBytes("val"));
event = event.reset(buffer, 0);
GenericRecord record = _decoder.getGenericRecord(event);
_eventHandler.onRecord(event, record);
count++;
}
_eventHandler.onEnd(count);
}
finally
{
DBHelper.close(rs,stmt,conn);
}
}
public Connection getConnection()
{
Connection conn = null;
if (_bootstrapConn == null)
{
LOG.info("<<<< Creating Bootstrap Connection!! >>>>");
_bootstrapConn = new BootstrapConn();
try
{
_bootstrapConn.initBootstrapConn(false,
_bsStaticConfig.getBootstrap().getBootstrapDBUsername(),
_bsStaticConfig.getBootstrap().getBootstrapDBPassword(),
_bsStaticConfig.getBootstrap().getBootstrapDBHostname(),
_bsStaticConfig.getBootstrap().getBootstrapDBName());
}
catch (Exception e)
{
LOG.fatal("Unable to open BootstrapDB Connection !!", e);
throw new RuntimeException("Got exception when getting bootstrap DB Connection.", e);
}
}
try
{
conn = _bootstrapConn.getDBConn();
}
catch (SQLException e)
{
LOG.fatal("Not able to open BootstrapDB Connection !!", e);
throw new RuntimeException("Got exception when getting bootstrap DB Connection.", e);
}
return conn;
}
public String getQuery()
{
String table = getTableName();
StringBuilder sql = new StringBuilder();
sql.append("select val from ").append(table)
.append(" where ").append(_queryStaticConfig.getQueryKey());
if (_queryStaticConfig.isRangeQuery())
{
sql.append(" >= ").append(_queryStaticConfig.getMinKey())
.append(" and ").append(_queryStaticConfig.getQueryKey())
.append(" <= ").append(_queryStaticConfig.getMaxKey());
}
else
{
sql.append(" = ").append(_queryStaticConfig.getValue());
}
return sql.toString();
}
public static void init(String[] args) throws Exception
{
parseArgs(args);
BootstrapSeederMain.Config bsConf = new BootstrapSeederMain.Config();
ConfigLoader<BootstrapSeederMain.StaticConfig> configLoader =
new ConfigLoader<BootstrapSeederMain.StaticConfig>("databus.reader.", bsConf);
_bsStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
Config qConf = new Config();
ConfigLoader<StaticConfig> configLoader2 =
new ConfigLoader<StaticConfig>("databus.query.", qConf);
_queryStaticConfig = configLoader2.loadConfig(_sQueryConfigProps);
SchemaRegistryService schemaRegistry =
FileSystemSchemaRegistryService.build(_bsStaticConfig.getSchemaRegistry().getFileSystem());
LOG.info("Schema = " + schemaRegistry.fetchLatestSchemaBySourceName(_queryStaticConfig.getSourceName()));
_schema = Schema.parse(schemaRegistry.fetchLatestSchemaBySourceName(_queryStaticConfig.getSourceName()));
VersionedSchema vs = new VersionedSchema(_schema.getFullName(), (short)1, _schema, null);
VersionedSchemaSet schemaSet = new VersionedSchemaSet();
schemaSet.add(vs);
_decoder = new DbusEventAvroDecoder(schemaSet);
}
public static Schema getSchema()
{
return _schema;
}
@SuppressWarnings("static-access")
public static void parseArgs(String[] args) throws IOException
{
CommandLineParser cliParser = new GnuParser();
Option helpOption = OptionBuilder.withLongOpt(HELP_OPT_LONG_NAME)
.withDescription("Help screen")
.create(HELP_OPT_CHAR);
Option sourcesOption = OptionBuilder.withLongOpt(QUERY_CONFIG_OPT_LONG_NAME)
.withDescription("Query Config")
.hasArg()
.withArgName("property_file")
.create(QUERY_CONFIG_OPT_CHAR);
Option dbOption = OptionBuilder.withLongOpt(BOOTSTRAP_DB_PROPS_OPT_LONG_NAME)
.withDescription("Bootstrap DB properties to use")
.hasArg()
.withArgName("property_file")
.create(BOOTSTRAP_DB_PROP_OPT_CHAR);
Option log4jPropsOption = OptionBuilder.withLongOpt(LOG4J_PROPS_OPT_LONG_NAME)
.withDescription("Log4j properties to use")
.hasArg()
.withArgName("property_file")
.create(LOG4J_PROPS_OPT_CHAR);
Options options = new Options();
options.addOption(helpOption);
options.addOption(sourcesOption);
options.addOption(dbOption);
options.addOption(log4jPropsOption);
CommandLine cmd = null;
try
{
cmd = cliParser.parse(options, args);
}
catch (ParseException pe)
{
LOG.fatal("Bootstrap Physical Config: failed to parse command-line options.", pe);
throw new RuntimeException("Bootstrap Physical Config: failed to parse command-line options.", pe);
}
if (cmd.hasOption(LOG4J_PROPS_OPT_CHAR))
{
String log4jPropFile = cmd.getOptionValue(LOG4J_PROPS_OPT_CHAR);
PropertyConfigurator.configure(log4jPropFile);
LOG.info("Using custom logging settings from file " + log4jPropFile);
}
else
{
PatternLayout defaultLayout = new PatternLayout("%d{ISO8601} +%r [%t] (%p) {%c} %m%n");
ConsoleAppender defaultAppender = new ConsoleAppender(defaultLayout);
Logger.getRootLogger().removeAllAppenders();
Logger.getRootLogger().addAppender(defaultAppender);
LOG.info("Using default logging settings");
}
if (cmd.hasOption(HELP_OPT_CHAR))
{
printCliHelp(options);
System.exit(0);
}
if (!cmd.hasOption(QUERY_CONFIG_OPT_CHAR))
{
throw new RuntimeException("Query Config is not provided; use --help for usage");
}
if (!cmd.hasOption(BOOTSTRAP_DB_PROP_OPT_CHAR))
{
throw new RuntimeException("Bootstrap config is not provided; use --help for usage");
}
String propFile1 = cmd.getOptionValue(QUERY_CONFIG_OPT_CHAR);
String propFile2 = cmd.getOptionValue(BOOTSTRAP_DB_PROP_OPT_CHAR);
LOG.info("Loading bootstrap DB config from properties file " + propFile2);
_sQueryConfigProps = new Properties();
FileInputStream f1 = new FileInputStream(propFile1);
try
{
_sQueryConfigProps.load(f1);
}
finally
{
f1.close();
}
_sBootstrapConfigProps = new Properties();
FileInputStream f2 = new FileInputStream(propFile2);
try
{
_sBootstrapConfigProps.load(f2);
}
finally
{
f2.close();
}
}
private static void printCliHelp(Options cliOptions)
{
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.printHelp("java " + BootstrapSeederMain.class.getName(), cliOptions);
}
public static class StaticConfig
{
private final Integer sourceId;
private final String sourceName;
private final String tablePrefix;
private final String queryKey;
private final boolean isRangeQuery;
private final Long value;
private final Long minKey;
private final Long maxKey;
private final Long logId;
public Long getLogId()
{
return logId;
}
public Integer getSourceId()
{
return sourceId;
}
public String getSourceName()
{
return sourceName;
}
public String getTablePrefix()
{
return tablePrefix;
}
public String getQueryKey()
{
return queryKey;
}
public boolean isRangeQuery()
{
return isRangeQuery;
}
public Long getValue()
{
return value;
}
public Long getMinKey()
{
return minKey;
}
public Long getMaxKey()
{
return maxKey;
}
public StaticConfig(Integer sourceId, String sourceName, String tablePrefix,
String queryKey, boolean isRangeQuery, Long value, Long minKey,
Long maxKey, Long logId)
{
super();
this.sourceId = sourceId;
this.sourceName = sourceName;
this.tablePrefix = tablePrefix;
this.queryKey = queryKey;
this.isRangeQuery = isRangeQuery;
this.value = value;
this.minKey = minKey;
this.maxKey = maxKey;
this.logId = logId;
}
@Override
public String toString()
{
return "StaticConfig [" +
"sourceId=" + sourceId +
", sourceName=" + sourceName +
", tablePrefix=" + tablePrefix +
", queryKey=" + queryKey +
", isRangeQuery=" + isRangeQuery +
", value=" + value +
", minKey=" + minKey +
", maxKey=" + maxKey +
"]";
}
}
public static class Config implements ConfigBuilder<StaticConfig>
{
private Integer sourceId;
private String sourceName;
private String tablePrefix;
private String field;
private boolean isRangeQuery;
private Long value;
private Long minValue;
private Long maxValue;
private Long logId;
public Long getLogId()
{
return logId;
}
public void setLogId(Long logId)
{
this.logId = logId;
}
public String getSourceName()
{
return sourceName;
}
public void setSourceName(String sourceName)
{
this.sourceName = sourceName;
}
public Integer getSourceId() {
return sourceId;
}
public void setSourceId(Integer sourceId)
{
this.sourceId = sourceId;
}
public String getTablePrefix()
{
return tablePrefix;
}
public void setTablePrefix(String tablePrefix)
{
this.tablePrefix = tablePrefix;
}
public String getField()
{
return field;
}
public void setField(String queryKey)
{
this.field = queryKey;
}
public boolean isRangeQuery()
{
return isRangeQuery;
}
public void setRangeQuery(boolean isRangeQuery)
{
this.isRangeQuery = isRangeQuery;
}
public Long getValue()
{
return value;
}
public void setValue(Long value)
{
this.value = value;
}
public Long getMinValue()
{
return minValue;
}
public void setMinValue(Long minKey)
{
this.minValue = minKey;
}
public Long getMaxValue()
{
return maxValue;
}
public void setMaxValue(Long maxKey)
{
this.maxValue = maxKey;
}
@Override
public StaticConfig build() throws InvalidConfigException
{
return new StaticConfig(sourceId, sourceName, tablePrefix, field, isRangeQuery, value, minValue, maxValue, logId);
}
}
}