/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.addthis.hydra.job.store;
import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import com.addthis.basis.util.Parameter;
import com.addthis.hydra.query.spawndatastore.AliasBiMap;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.addthis.hydra.job.store.SpawnDataStoreKeys.SPAWN_BALANCE_PARAM_PATH;
import static com.addthis.hydra.job.store.SpawnDataStoreKeys.SPAWN_COMMON_ALERT_PATH;
import static com.addthis.hydra.job.store.SpawnDataStoreKeys.SPAWN_COMMON_COMMAND_PATH;
import static com.addthis.hydra.job.store.SpawnDataStoreKeys.SPAWN_COMMON_MACRO_PATH;
import static com.addthis.hydra.job.store.SpawnDataStoreKeys.SPAWN_HOST_FAIL_WORKER_PATH;
import static com.addthis.hydra.job.store.SpawnDataStoreKeys.SPAWN_JOB_CONFIG_PATH;
import static com.addthis.hydra.job.store.SpawnDataStoreKeys.SPAWN_QUEUE_PATH;
/**
* A class providing various utility functions for common data storage between hydra components
*/
public class DataStoreUtil {
/* A list of datastore paths with values that should be cutover */
private static final List<String> pathsToImport = Arrays.asList(SPAWN_QUEUE_PATH, SPAWN_BALANCE_PARAM_PATH, SPAWN_HOST_FAIL_WORKER_PATH);
/* A list of datastore parent nodes with children that should be cutover */
private static final List<String> parentsToImport = Arrays.asList(SPAWN_COMMON_COMMAND_PATH, SPAWN_COMMON_MACRO_PATH, SPAWN_JOB_CONFIG_PATH, AliasBiMap.ALIAS_PATH, SPAWN_COMMON_ALERT_PATH);
/* A list of nodes beneath each job node */
private static final List<String> jobParametersToImport = Arrays.asList("config", "queryconfig", "tasks", "alerts");
/* A list of properties of certain job nodes that should be imported as flat values rather than children -- necessary for certain kafka broker info */
private static final List<String> jobParameterToImportFlat = Arrays.asList("brokerinfo");
private static final Logger log = LoggerFactory.getLogger(DataStoreUtil.class);
/* The following parameters _must_ be the same for the mqmaster and spawn process. */
private static final String canonicalDataStoreType = Parameter.value("spawn.datastore.type", "ZK");
private static final String cutoverOnceFromType = Parameter.value("spawn.datastore.cutoverOnceFromType");
private static final int numCutoverThreads = Parameter.intValue("spawn.datastore.numCutoverThreads", 5);
private static final int cutoverTimeoutMinutes = Parameter.intValue("spawn.datstore.cutoverTimeoutMinutes", 15);
private static final String clusterName = Parameter.value("cluster.name", "localhost");
private static final String sqlTableName = Parameter.value("spawn.sql.table", "sdsTable_" + clusterName);
private static final String sqlDbName = Parameter.value("spawn.sql.db", "sdsDB_" + clusterName);
private static final String sqlHostName = Parameter.value("spawn.sql.host", "localhost");
private static final String sqlUser = Parameter.value("spawn.sql.user"); // Intentionally defaults to null for no user/pass
private static final String sqlPassword = Parameter.value("spawn.sql.password", "");
private static final int sqlPort = Parameter.intValue("spawn.sql.port", 3306);
private static final String markCutoverCompleteKey = "/spawndatastore.cutover.complete";
private static final String fsDataStoreFileRoot = Parameter.value("spawn.datastore.fs.dir", "etc/datastore");
public static enum DataStoreType {ZK, MYSQL, FS, POSTGRES}
/**
* Create the canonical SpawnDataStore based on the system parameters
*
* @return A SpawnDataStore of the appropriate implementation
*/
public static SpawnDataStore makeCanonicalSpawnDataStore() throws Exception {
return makeCanonicalSpawnDataStore(false);
}
/**
* Create the canonical SpawnDataStore after possibly first cutting over from an old type. After this cutover is
* successfully completed once, it will never be done again. Spawn owns the datastore cutover process, so only it
* uses cutoverIfNecessary=true.
*
* @param cutoverIfNecessary Whether to check whether a datastore cutover should be done
* @return The canonical data store
* @throws Exception If data store creation fails
*/
public static SpawnDataStore makeCanonicalSpawnDataStore(boolean cutoverIfNecessary) throws Exception {
SpawnDataStore canonicalDataStore = makeSpawnDataStore(DataStoreType.valueOf(canonicalDataStoreType));
if (cutoverIfNecessary && cutoverOnceFromType != null && canonicalDataStore.get(markCutoverCompleteKey) == null) {
SpawnDataStore oldDataStore = makeSpawnDataStore(DataStoreType.valueOf(cutoverOnceFromType));
cutoverBetweenDataStore(oldDataStore, canonicalDataStore, true);
canonicalDataStore.put(markCutoverCompleteKey, "1");
}
return canonicalDataStore;
}
public static SpawnDataStore makeSpawnDataStore(DataStoreType type) throws Exception {
Properties properties = new Properties();
if (sqlUser != null) {
properties.put("user", sqlUser);
properties.put("password", sqlPassword);
}
switch (type) {
case FS: return new FilesystemDataStore(new File(fsDataStoreFileRoot));
case ZK: return new ZookeeperDataStore(null);
case MYSQL:
return new MysqlDataStore("jdbc:mysql:thin://" + sqlHostName + ":" + sqlPort + "/", sqlDbName, sqlTableName, properties);
case POSTGRES:
return new PostgresDataStore("jdbc:postgres://" + sqlHostName + ":" + sqlPort + "/", sqlDbName, sqlTableName, properties);
default: throw new IllegalArgumentException("Unexpected DataStoreType " + type);
}
}
/**
* A method to cut over all necessary data from on DataStore to another. Placeholder for now until more data stores are implemented.
*
* @param sourceDataStore The old datastore to read from
* @param targetDataStore The new datastore to push data to
* @throws Exception If any part of the cutover fails
*/
public static void cutoverBetweenDataStore(SpawnDataStore sourceDataStore, SpawnDataStore targetDataStore, boolean checkAllWrites) throws Exception {
log.warn("Beginning cutover from {} to {}", sourceDataStore.getDescription(), targetDataStore.getDescription());
for (String path : pathsToImport) {
importValue(path, sourceDataStore, targetDataStore, checkAllWrites);
}
for (String parent : parentsToImport) {
importParentAndChildren(parent, sourceDataStore, targetDataStore, checkAllWrites);
}
List<String> jobIds = sourceDataStore.getChildrenNames(SPAWN_JOB_CONFIG_PATH);
if (jobIds != null) {
if (numCutoverThreads <= 1) {
for (String jobId : jobIds) {
importJobData(jobId, sourceDataStore, targetDataStore, checkAllWrites);
}
} else {
importJobDataParallel(jobIds, sourceDataStore, targetDataStore, checkAllWrites);
}
}
log.warn("Finished cutover from {} to {}", sourceDataStore.getDescription(), targetDataStore.getDescription());
}
private static void importJobDataParallel(List<String> jobIds, SpawnDataStore sourceDataStore, SpawnDataStore targetDataStore, boolean checkAllWrites) throws Exception {
ExecutorService executorService = new ThreadPoolExecutor(numCutoverThreads, numCutoverThreads, 60, TimeUnit.SECONDS,
new LinkedBlockingQueue<>());
AtomicBoolean gotFailures = new AtomicBoolean(false);
int partitionSize = Math.max((int) Math.ceil((double) jobIds.size() / numCutoverThreads), 1);
for (List<String> partition : Lists.partition(jobIds, partitionSize)) {
executorService.submit(new CutoverWorker(sourceDataStore, targetDataStore, gotFailures, partition, checkAllWrites));
}
executorService.shutdown();
executorService.awaitTermination(cutoverTimeoutMinutes, TimeUnit.MINUTES);
if (gotFailures.get()) {
throw new RuntimeException("A cutover worker has failed; see log for details");
}
}
private static class CutoverWorker implements Runnable {
SpawnDataStore sourceDataStore;
SpawnDataStore targetDataStore;
private AtomicBoolean gotFailures;
private List<String> jobIdPartition;
private boolean checkAllWrites;
private CutoverWorker(SpawnDataStore sourceDataStore, SpawnDataStore targetDataStore, AtomicBoolean gotFailures, List<String> jobIdPartition, boolean checkAllWrites) {
this.sourceDataStore = sourceDataStore;
this.targetDataStore = targetDataStore;
this.gotFailures = gotFailures;
this.jobIdPartition = jobIdPartition;
this.checkAllWrites = checkAllWrites;
}
@Override
public void run() {
try {
for (String jobId : jobIdPartition) {
importJobData(jobId, sourceDataStore, targetDataStore, checkAllWrites);
}
} catch (Exception ex) {
gotFailures.compareAndSet(false, true);
log.error("Exception during datastore cutover", ex);
}
}
}
private static void importJobData(String jobId, SpawnDataStore sourceDataStore, SpawnDataStore targetDataStore, boolean checkAllWrites) throws Exception {
log.debug("Cutting over job data for {}", jobId);
String basePath = SPAWN_JOB_CONFIG_PATH + "/" + jobId;
importValue(basePath, sourceDataStore, targetDataStore, checkAllWrites);
for (String parameter : jobParametersToImport) {
importValue(basePath + "/" + parameter, sourceDataStore, targetDataStore, checkAllWrites);
}
for (String flatParameter : jobParameterToImportFlat) {
String path = basePath + "/" + flatParameter;
if (sourceDataStore.get(path) != null) {
Map<String, String> children = sourceDataStore.getAllChildren(path);
for (Map.Entry<String, String> child : children.entrySet()) {
targetDataStore.put(path + "/" + child.getKey(), child.getValue());
}
}
}
}
/**
* Internal function to import the value of a row from one datastore to another
*
* @param path The path to import
* @param sourceDataStore The source to read from
* @param targetDataStore The target to write to
* @throws Exception If there is a problem during the transfer
*/
private static void importValue(String path, SpawnDataStore sourceDataStore, SpawnDataStore targetDataStore, boolean checkAllWrites) throws Exception {
log.debug("Cutting over value of path {}", path);
String sourceValue = sourceDataStore.get(path);
if (sourceValue != null) {
targetDataStore.put(path, sourceValue);
if (checkAllWrites) {
String checkedValue = targetDataStore.get(path);
if (!sourceValue.equals(checkedValue)) {
throw new RuntimeException("INCORRECT TARGET VALUE DETECTED FOR PATH " + path);
}
}
}
}
/**
* Internal function to important all children beneath a certain parent path from one datastore to another
*
* @param parent The parent id to grab the children from
* @param sourceDataStore The source to read from
* @param targetDataStore The target to write to
* @throws Exception If there is a problem during the transfer
*/
private static void importParentAndChildren(String parent, SpawnDataStore sourceDataStore, SpawnDataStore targetDataStore, boolean checkAllWrites) throws Exception {
log.debug("Cutting over children of path {}", parent);
importValue(parent, sourceDataStore, targetDataStore, checkAllWrites);
List<String> children = sourceDataStore.getChildrenNames(parent);
if (children == null) {
return;
}
for (String child : children) {
String sourceValue = sourceDataStore.getChild(parent, child);
if (sourceValue != null) {
targetDataStore.putAsChild(parent, child, sourceValue);
if (checkAllWrites) {
String childValue = targetDataStore.getChild(parent, child);
if (!sourceValue.equals(childValue)) {
throw new RuntimeException("INCORRECT CHILD VALUE DETECTED FOR PATH " + parent + " CHILD " + child);
}
}
}
}
}
}