/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.addthis.hydra.query.spawndatastore;
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;
import java.util.concurrent.ExecutionException;
import com.addthis.basis.util.Parameter;
import com.addthis.bark.StringSerializer;
import com.addthis.hydra.job.JobQueryConfig;
import com.addthis.hydra.job.store.AvailableCache;
import com.addthis.hydra.job.store.SpawnDataStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.addthis.codec.json.CodecJSON.INSTANCE;
import static com.addthis.hydra.job.store.SpawnDataStoreKeys.SPAWN_JOB_CONFIG_PATH;
/**
* A class to watch job config data within the SpawnDataStore, cache it, and serve information relevant to
* query functions
*/
@ThreadSafe
public class QueryConfigWatcher {
private static final Logger log = LoggerFactory.getLogger(QueryConfigWatcher.class);
/* How long job config data can live in the cache before being refreshed */
private static final long QUERY_CONFIG_REFRESH_MILLIS = Parameter.longValue("query.config.refresh.millis", 15000);
/* How many jobIds should be stored in the cache */
private static final int QUERY_CONFIG_CACHE_SIZE = Parameter.intValue("query.config.cache.size", 100);
/* A SpawnDataStore used to fetch the config data. Should be the same type (zookeeper/priam)
as the one used by Spawn to store job data */
private final SpawnDataStore spawnDataStore;
/* A LoadingCache used to save configs fetched from the SpawnDataStore */
private final AvailableCache<JobQueryConfig> configCache;
public QueryConfigWatcher(SpawnDataStore spawnDataStore) {
this.spawnDataStore = spawnDataStore;
// This cache will not block on queryconfig fetches unless no data has been fetched for that job before
this.configCache =
new AvailableCache<JobQueryConfig>(QUERY_CONFIG_REFRESH_MILLIS, -1, QUERY_CONFIG_CACHE_SIZE, 2) {
@Nullable @Override public JobQueryConfig fetchValue(String id) {
JobQueryConfig jobQueryConfig = new JobQueryConfig();
String jobConfigPath = SPAWN_JOB_CONFIG_PATH + "/" + id + "/queryconfig";
String raw = QueryConfigWatcher.this.spawnDataStore.get(jobConfigPath);
if (raw == null) {
return null;
}
try {
INSTANCE.decode(jobQueryConfig, StringSerializer.deserialize(raw.getBytes()).getBytes());
return jobQueryConfig;
} catch (Exception e) {
log.warn("Failed to decode query config", e);
return null;
}
}
};
}
/**
* Fetch the query config for a jobId, using the cached result if possible
*
* @param jobId The jobId to fetch
* @return A (possibly null) JobQueryConfig
*/
@Nullable public JobQueryConfig getJobQueryConfig(String jobId) {
try {
return configCache.get(jobId);
} catch (ExecutionException e) {
log.warn("Exception during JobQueryConfig fetch", e);
// JobQueryConfig was not found, return null
return null;
}
}
/** Throw away the contents of the cache, primarily for testing */
public void invalidateQueryConfigCache() {
configCache.clear();
}
/**
* Check whether a job is safe to query, using the cache if possible
*
* @param jobID The jobId to check
* @return True if the job is queryable
*/
public boolean safeToQuery(String jobID) {
JobQueryConfig jqc = getJobQueryConfig(jobID);
return (jqc != null) && jqc.getCanQuery();
}
}