/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.addthis.hydra.query.spawndatastore; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import com.addthis.hydra.data.query.QueryException; import com.addthis.hydra.job.IJob; import com.addthis.hydra.job.Job; import com.addthis.hydra.job.JobConfigManager; import com.addthis.hydra.job.store.DataStoreUtil; import com.addthis.hydra.job.store.SpawnDataStore; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; public class SpawnDataStoreHandler implements AutoCloseable { /** * A ZooKeeper/Priam backed data structure that keeps track of * the Hydra jobs in the cluster. We are specifically * interested in how many tasks are in each job so we know * the minimum number of responses required to run a query. */ private final JobConfigManager jobConfigManager; /** * A SpawnDataStore used by the JobConfigManager to fetch job status. * Uses either zookeeper or priam depending on the cluster config. */ private final SpawnDataStore spawnDataStore; /** * Monitors ZooKeeper for query configuration information. * Used to determine if queries are enabled and/or traceable * for a given job */ private final QueryConfigWatcher queryConfigWatcher; /** * A ZooKeeper backed data structure that maintains a * bi-directional mapping of job aliases to job IDs */ private final AliasBiMap aliasBiMap; /** * a cache of job configuration data, used to reduce load placed on ZK server with high volume queries */ private final LoadingCache<String, IJob> jobConfigurationCache = CacheBuilder .newBuilder() .maximumSize(2000) .refreshAfterWrite(10, TimeUnit.MINUTES) .build(new CacheLoader<String, IJob>() { @Override public IJob load(String key) { return jobConfigManager.getJob(key); } }); public SpawnDataStoreHandler() throws Exception { spawnDataStore = DataStoreUtil.makeCanonicalSpawnDataStore(); this.jobConfigManager = new JobConfigManager(spawnDataStore); this.queryConfigWatcher = new QueryConfigWatcher(spawnDataStore); this.aliasBiMap = new AliasBiMap(spawnDataStore); this.aliasBiMap.loadCurrentValues(); } @Override public void close() { spawnDataStore.close(); } public void validateJobForQuery(String job) { if (!queryConfigWatcher.safeToQuery(job)) { throw new QueryException("job is not safe to query (are queries enabled for this job in spawn?): " + job); } } public List<String> expandAlias(String job) { List<String> possibleJobs = aliasBiMap.getJobs(job); if ((possibleJobs != null) && !possibleJobs.isEmpty()) { return possibleJobs; } return Collections.singletonList(job); } public String resolveAlias(String job) { List<String> possibleJobs = aliasBiMap.getJobs(job); if ((possibleJobs != null) && !possibleJobs.isEmpty()) { return possibleJobs.get(0); } return job; } public int getCononicalTaskCount(String job) { IJob zkJob; try { zkJob = jobConfigurationCache.get(job); } catch (ExecutionException ignored) { throw new QueryException("unable to retrieve job configuration for job: " + job); } if (zkJob == null) { throw new QueryException("[MeshQueryMaster] Error: unable to find ZK reference for job: " + job); } return new Job(zkJob).getTaskCount(); } }