package com.linkedin.databus2.client.util;
/*
*
* Copyright 2013 LinkedIn Corp. All rights reserved
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
import java.io.FileReader;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.map.JsonMappingException;
import com.linkedin.databus.client.DatabusHttpClientImpl;
import com.linkedin.databus.client.DatabusHttpClientImpl.CheckpointPersistenceStaticConfig;
import com.linkedin.databus.client.DatabusHttpClientImpl.CheckpointPersistenceStaticConfigBuilder;
import com.linkedin.databus.client.pub.CheckpointPersistenceProvider;
import com.linkedin.databus.client.pub.FileSystemCheckpointPersistenceProvider;
import com.linkedin.databus.core.BootstrapCheckpointHandler;
import com.linkedin.databus.core.Checkpoint;
import com.linkedin.databus.core.DatabusRuntimeException;
import com.linkedin.databus.core.DbusClientMode;
import com.linkedin.databus.core.util.ConfigLoader;
/** Utility that can be used to serialize a checkpoint to a file */
//TODO We have to rethink this class and see if it is needed anymore. We also have to figure out what is the right level
//of abstraction as currently it is too low level and needs access to internal checkpoint methods.
public class CheckpointSerializerMain
{
public static final String MODULE = CheckpointSerializerMain.class.getSimpleName();
public static final Logger LOG = Logger.getLogger(MODULE);
public static final String HELP_OPT_NAME = "help";
public static final String HELP_OPT_DESCR = "prints this help screen";
public static final String ACTION_OPT_NAME = "action";
public static final String ACTION_OPT_DESCR = "action to run: PRINT, CHANGE, DELETE";
public static final String CLIENT_PROPS_FILE_OPT_NAME = "client_props";
public static final String CLIENT_PROPS_FILE_OPT_DESCR = "specifies a client configuration properties file";
public static final String CP3_PROPS_FILE_OPT_NAME = "cp3_props";
public static final String CP3_PROPS_FILE_OPT_DESCR = "specifies a checkpoint persistence provider (CP3) configuration properties file ";
public static final String PROPS_PREFIX_OPT_NAME = "props_prefix";
public static final String PROPS_PREFIX_OPT_DESCR = "properties name prefix for the configuration file (client or CP3)";
public static final String SOURCES_OPT_NAME = "sources";
public static final String SOURCES_OPT_DESCR = "comma-separated source list for the checkpoint; -1 for flexible checkpoint";
public static final String SCN_OPT_NAME = "sequence_num";
public static final String SCN_OPT_DESCR = "new scn for the checkpoint to be saved";
public static final String TYPE_OPT_NAME = "type";
public static final String TYPE_OPT_DESCR = "the type of the checkpoint to be saved: BOOTSTRAP_SNAPSHOT, BOOTSTRAP_CATCHUP, ONLINE_CONSUMPTION";
public static final String SINCE_SCN_OPT_NAME = "since_sequence_num";
public static final String SINCE_SCN_OPT_DESCR = "sequence number for when the bootstrap started (snapshot and catchup)";
public static final String START_SCN_OPT_NAME = "start_sequence_num";
public static final String START_SCN_OPT_DESCR = "start sequence number for bootstrap checkpoints (snapshot and catchup)";
public static final String TARGET_SCN_OPT_NAME = "target_sequence_num";
public static final String TARGET_SCN_OPT_DESCR = "target sequence number for bootstrap checkpoints (catchup only)";
public static final String BOOTSTRAP_SOURCE_OPT_NAME = "bootstrap_source";
public static final String BOOTSTRAP_SOURCE_OPT_DESCR = "the current bootstrap source name";
public static final char ACTION_OPT_CHAR = 'a';
public static final char BOOTSTRAP_SOURCE_OPT_CHAR = 'b';
public static final char CLIENT_PROPS_FILE_OPT_CHAR = 'c';
public static final char CP3_PROPS_FILE_OPT_CHAR = 'C';
public static final char PROPS_PREFIX_OPT_CHAR = 'f';
public static final char HELP_OPT_CHAR = 'h';
public static final char SOURCES_OPT_CHAR = 'S';
public static final char SCN_OPT_CHAR = 's';
public static final char TYPE_OPT_CHAR = 't';
public static final char SINCE_SCN_OPT_CHAR = 'x';
public static final char START_SCN_OPT_CHAR = 'y';
public static final char TARGET_SCN_OPT_CHAR = 'z';
private static enum Action
{
PRINT,
CHANGE,
DELETE
}
private static Action _action;
private static String[] _sources;
private static Properties _clientProps;
private static Properties _cp3Props;
private static String _propPrefix;
private static Long _scn;
private static Long _sinceScn;
private static Long _startScn;
private static Long _targetScn;
private static DbusClientMode _cpType;
private static String _bootstrapSource;
@SuppressWarnings("static-access")
private static Options createOptions()
{
Option helpOption = OptionBuilder.withLongOpt(HELP_OPT_NAME)
.withDescription(HELP_OPT_DESCR)
.create(HELP_OPT_CHAR);
Option clientPropsOption = OptionBuilder.withLongOpt(CLIENT_PROPS_FILE_OPT_NAME)
.withDescription(CLIENT_PROPS_FILE_OPT_DESCR)
.hasArg()
.withArgName("properties_file")
.create(CLIENT_PROPS_FILE_OPT_CHAR);
Option cp3PropsOption = OptionBuilder.withLongOpt(CP3_PROPS_FILE_OPT_NAME)
.withDescription(CP3_PROPS_FILE_OPT_DESCR)
.hasArg()
.withArgName("properties_file")
.create(CP3_PROPS_FILE_OPT_CHAR);
Option propsPrefixOption = OptionBuilder.withLongOpt(PROPS_PREFIX_OPT_NAME)
.withDescription(PROPS_PREFIX_OPT_DESCR)
.hasArg()
.withArgName("prefix_string")
.create(PROPS_PREFIX_OPT_CHAR);
Option sourcesOption = OptionBuilder.withLongOpt(SOURCES_OPT_NAME)
.withDescription(SOURCES_OPT_DESCR)
.hasArg()
.withArgName("sources_list")
.create(SOURCES_OPT_CHAR);
Option scnOptOption = OptionBuilder.withLongOpt(SCN_OPT_NAME)
.withDescription(SCN_OPT_DESCR)
.hasArg()
.withArgName("sequence_number")
.create(SCN_OPT_CHAR);
Option actionOption = OptionBuilder.withLongOpt(ACTION_OPT_NAME)
.withDescription(ACTION_OPT_DESCR)
.hasArg()
.withArgName("action")
.create(ACTION_OPT_CHAR);
Option sinceScnOptOption = OptionBuilder.withLongOpt(SINCE_SCN_OPT_NAME)
.withDescription(SINCE_SCN_OPT_DESCR)
.hasArg()
.withArgName("sequence_number")
.create(SINCE_SCN_OPT_CHAR);
Option startScnOptOption = OptionBuilder.withLongOpt(START_SCN_OPT_NAME)
.withDescription(START_SCN_OPT_DESCR)
.hasArg()
.withArgName("sequence_number")
.create(START_SCN_OPT_CHAR);
Option targetScnOptOption = OptionBuilder.withLongOpt(TARGET_SCN_OPT_NAME)
.withDescription(TARGET_SCN_OPT_DESCR)
.hasArg()
.withArgName("sequence_number")
.create(TARGET_SCN_OPT_CHAR);
Option typeOption = OptionBuilder.withLongOpt(TYPE_OPT_NAME)
.withDescription(TYPE_OPT_DESCR)
.hasArg()
.withArgName("checkpoint_type")
.create(TYPE_OPT_CHAR);
Option bootstrapSourceOption = OptionBuilder.withLongOpt(BOOTSTRAP_SOURCE_OPT_NAME)
.withDescription(BOOTSTRAP_SOURCE_OPT_DESCR)
.hasArg()
.withArgName("bootstrap_source_name")
.create(BOOTSTRAP_SOURCE_OPT_CHAR);
Options options = new Options();
options.addOption(helpOption);
options.addOption(actionOption);
options.addOption(clientPropsOption);
options.addOption(cp3PropsOption);
options.addOption(propsPrefixOption);
options.addOption(sourcesOption);
options.addOption(scnOptOption);
options.addOption(sinceScnOptOption);
options.addOption(startScnOptOption);
options.addOption(targetScnOptOption);
options.addOption(typeOption);
options.addOption(bootstrapSourceOption);
return options;
}
private static void parseArgs(String[] args) throws Exception
{
CommandLineParser cliParser = new GnuParser();
Options options = createOptions();
CommandLine cmd = null;
try
{
cmd = cliParser.parse(options, args);
}
catch (ParseException pe)
{
throw new RuntimeException("failed to parse command-line options.", pe);
}
if (cmd.hasOption(HELP_OPT_CHAR) || 0 == cmd.getOptions().length)
{
printCliHelp(options);
System.exit(0);
}
try
{
_action = Action.valueOf(cmd.getOptionValue(ACTION_OPT_CHAR).toUpperCase());
}
catch (Exception e)
{
throw new RuntimeException("invalid action: " + cmd.getOptionValue(ACTION_OPT_CHAR), e);
}
if (! cmd.hasOption(SOURCES_OPT_CHAR))
{
throw new RuntimeException("expected sources list; see --help for more info");
}
String sourcesListStr = cmd.getOptionValue(SOURCES_OPT_CHAR);
_sources = sourcesListStr.split(",");
if (null == _sources || 0 == _sources.length)
{
throw new RuntimeException("empty sources list");
}
for (int i = 0; i < _sources.length; ++i) _sources[i] = _sources[i].trim();
if (Action.PRINT != _action && ! cmd.hasOption(CLIENT_PROPS_FILE_OPT_CHAR) &&
! cmd.hasOption(CP3_PROPS_FILE_OPT_CHAR))
{
throw new RuntimeException("expected client or CP3 configuration; see --help for more info");
}
String defaultPropPrefix = null;
if (cmd.hasOption(CLIENT_PROPS_FILE_OPT_CHAR))
{
try
{
_clientProps = loadProperties(cmd.getOptionValue(CLIENT_PROPS_FILE_OPT_CHAR));
defaultPropPrefix = "databus2.client";
}
catch (Exception e)
{
throw new RuntimeException("unable to load client properties", e);
}
}
else if (cmd.hasOption(CP3_PROPS_FILE_OPT_CHAR))
{
try
{
_cp3Props = loadProperties(cmd.getOptionValue(CP3_PROPS_FILE_OPT_CHAR));
defaultPropPrefix = "databus2.client.checkpointPersistence";
}
catch (Exception e)
{
throw new RuntimeException("unable to load CP3 properties", e);
}
}
_propPrefix = cmd.hasOption(PROPS_PREFIX_OPT_CHAR) ? cmd.getOptionValue(PROPS_PREFIX_OPT_CHAR)
: defaultPropPrefix;
if (null != _propPrefix && ! _propPrefix.endsWith("."))
{
_propPrefix = _propPrefix + ".";
}
if (! cmd.hasOption(ACTION_OPT_CHAR))
{
throw new RuntimeException("action expected; see --help for more info");
}
_scn = parseLongOption(cmd, SCN_OPT_CHAR, "sequence number");
_sinceScn = parseLongOption(cmd, SINCE_SCN_OPT_CHAR, "last sequence number");
_startScn = parseLongOption(cmd, START_SCN_OPT_CHAR, "start sequence number");
_targetScn = parseLongOption(cmd, TARGET_SCN_OPT_CHAR, "target sequence number");
if (cmd.hasOption(TYPE_OPT_CHAR))
{
try
{
_cpType = DbusClientMode.valueOf(cmd.getOptionValue(TYPE_OPT_CHAR).toUpperCase());
}
catch (Exception e)
{
throw new RuntimeException("invalid checkpoint type:" + cmd.getOptionValue(TYPE_OPT_CHAR),
e);
}
}
if (cmd.hasOption(BOOTSTRAP_SOURCE_OPT_CHAR))
{
_bootstrapSource = cmd.getOptionValue(BOOTSTRAP_SOURCE_OPT_CHAR);
}
}
private static Long parseLongOption(CommandLine cmd, char optionChar, String argName)
{
Long result = null;
if (cmd.hasOption(optionChar))
{
try
{
result = Long.parseLong(cmd.getOptionValue(optionChar));
}
catch (NumberFormatException nfe)
{
throw new RuntimeException("invalid " + argName + ": " + cmd.getOptionValue(optionChar), nfe);
}
}
return result;
}
private static Properties loadProperties(String fileName) throws IOException
{
Properties result = new Properties();
FileReader freader = new FileReader(fileName);
try
{
result.load(freader);
}
finally
{
freader.close();
}
return result;
}
private static void printCliHelp(Options cliOptions)
{
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.setWidth(120);
helpFormatter.printHelp("java " + CheckpointSerializerMain.class.getName(), cliOptions);
}
private static Checkpoint updateCheckpoint(Checkpoint cpOld)
throws JsonParseException, JsonMappingException, IOException
{
Checkpoint cpNew = null != cpOld ? new Checkpoint(cpOld.toString()) : new Checkpoint();
if (null != _scn)
{
if (-1L != _scn)
{
cpNew.setWindowScn(_scn);
cpNew.setWindowOffset(0);
}
else
{
cpNew.setFlexible();
}
}
if (null != _startScn)
{
cpNew.setBootstrapStartScn(_startScn);
}
if (null != _targetScn)
{
cpNew.setBootstrapTargetScn(_targetScn);
}
if (null != _cpType)
{
cpNew.setConsumptionMode(_cpType);
switch (_cpType)
{
case ONLINE_CONSUMPTION: cpNew.setWindowOffset(0); break;
/*
* TODO Disabling as the bootstrap checkpoint creation leaves out important
* information (e.g. catchup/snashot source index) out of the checkpoint
* and thus is incorrect. We have to figure out what types of bootstrap
* checkpoints it makes sense to create.
case BOOTSTRAP_CATCHUP:
{
if (null != _bootstrapSource) cpNew.setCatchupSource(_bootstrapSource);
cpNew.setCatchupOffset(-1);
break;
}*/
case BOOTSTRAP_SNAPSHOT:
{
BootstrapCheckpointHandler handler = new BootstrapCheckpointHandler(_sources);
cpNew = handler.createInitialBootstrapCheckpoint(cpNew, _sinceScn);
//if (null != _bootstrapSource) cpNew.setSnapshotSource(_bootstrapSource);
cpNew.setSnapshotOffset(-1);
break;
}
default:
throw new DatabusRuntimeException("unsupported checkpoint type: " + _cpType);
}
}
return cpNew;
}
public static void main(String[] args) throws Exception
{
parseArgs(args);
PatternLayout defaultLayout = new PatternLayout("%d{ISO8601} +%r [%t] (%p) {%c{1}} %m%n");
ConsoleAppender defaultAppender = new ConsoleAppender(defaultLayout);
Logger.getRootLogger().removeAllAppenders();
Logger.getRootLogger().addAppender(defaultAppender);
Logger.getRootLogger().setLevel(Level.INFO);
Logger.getRootLogger().info("NOTE. This tool works only with V2/V1 checkpoints");
CheckpointPersistenceProvider cp3 = null;
if (null != _cp3Props)
{
CheckpointPersistenceStaticConfigBuilder cp3ConfBuilder =
new CheckpointPersistenceStaticConfigBuilder();
ConfigLoader<CheckpointPersistenceStaticConfig> configLoader =
new ConfigLoader<DatabusHttpClientImpl.CheckpointPersistenceStaticConfig>(
_propPrefix,
cp3ConfBuilder);
configLoader.loadConfig(_cp3Props);
CheckpointPersistenceStaticConfig cp3Conf = cp3ConfBuilder.build();
if (cp3Conf.getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM)
{
throw new RuntimeException("don't know what to do with cp3 type:" + cp3Conf.getType());
}
cp3 = new FileSystemCheckpointPersistenceProvider(cp3Conf.getFileSystem(), 2);
}
else if (null != _clientProps)
{
DatabusHttpClientImpl.Config clientConfBuilder =
new DatabusHttpClientImpl.Config();
ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader =
new ConfigLoader<DatabusHttpClientImpl.StaticConfig>(_propPrefix, clientConfBuilder);
configLoader.loadConfig(_clientProps);
DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
if (clientConf.getCheckpointPersistence().getType() !=
CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM)
{
throw new RuntimeException("don't know what to do with cp3 type:" +
clientConf.getCheckpointPersistence().getType());
}
cp3 = new FileSystemCheckpointPersistenceProvider(
clientConf.getCheckpointPersistence().getFileSystem(), 2);
}
List<String> sourceList = Arrays.asList(_sources);
Checkpoint cpOld = null != cp3 ? cp3.loadCheckpoint(sourceList) : new Checkpoint();
Checkpoint cpNew;
if (Action.PRINT == _action)
{
cpNew = updateCheckpoint(cpOld);
}
else if (Action.CHANGE == _action)
{
cpNew = updateCheckpoint(cpOld);
cp3.storeCheckpoint(sourceList, cpNew);
//reread as a sanity check
cpNew = cp3.loadCheckpoint(sourceList);
}
else if (Action.DELETE == _action)
{
cp3.removeCheckpoint(sourceList);
cpNew = cp3.loadCheckpoint(sourceList);
}
else
{
throw new RuntimeException("don't know what to do with action: " + _action);
}
if (null != cpOld) System.out.println("old: " + cpOld.toString());
else System.out.println("old: null");
if (null != cpNew) System.out.println("new: " + cpNew.toString());
else System.out.println("new: null");
}
}