/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.addthis.hydra.job.spawn.search;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.addthis.codec.jackson.Jackson;
import com.addthis.hydra.job.Job;
import com.addthis.hydra.job.JobConfigManager;
import com.addthis.hydra.job.JobParameter;
import com.addthis.hydra.job.entity.JobMacro;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.fasterxml.jackson.core.JsonGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.addthis.hydra.job.spawn.search.IncludeLocations.forMacros;
/**
* Searches job configurations based on the search options passed to the constructor.
*/
public class JobSearcher implements Runnable {
private static final Logger log = LoggerFactory.getLogger(JobSearcher.class);
private final Pattern pattern;
private final Map<String, JobMacro> macros;
private final Map<String, Job> jobs;
private final JobConfigManager jobConfigManager;
private final JsonGenerator generator;
private final Map<String, List<String>> aliases;
// job id -> macros included directly or indirectly in the job
private final Map<String, Set<String>> jobMacrosMap;
public JobSearcher(Map<String, Job> jobs,
Map<String, JobMacro> macros,
Map<String, List<String>> aliases,
JobConfigManager jobConfigManager,
SearchOptions options,
OutputStream outputStream) throws IOException {
this.jobs = jobs;
this.macros = macros;
this.aliases = aliases;
this.jobConfigManager = jobConfigManager;
this.pattern = Pattern.compile(options.pattern);
this.generator = Jackson.defaultMapper().getFactory().createGenerator(outputStream);
this.jobMacrosMap = new HashMap<>();
}
@Override
public void run() {
try {
generator.writeStartObject();
JobMacroGraph dependencyGraph = new JobMacroGraph(macros);
Map<String, Set<TextLocation>> macroSearches = searchMacros(macros, dependencyGraph);
/*
{
jobs: [
{id, description, matches: []}
]
macros: [
{id, description, matches: []}
]
}
*/
generator.writeArrayFieldStart("jobs");
for (Job job : jobs.values()) {
SearchResult jobSearchResult = searchJob(job, dependencyGraph, macroSearches);
if (jobSearchResult != null) {
generator.writeObject(jobSearchResult);
}
}
generator.writeEndArray();
generator.writeObjectField("macros", getMacroSearchResults(macroSearches));
generator.writeEndObject();
} catch (Exception e) {
log.error("JobSearcher failed:", e);
} finally {
try {
generator.close();
} catch (IOException e) {
log.error("JobSearcher generator failed to close", e);
}
}
}
@Nullable
private SearchResult searchJob(Job job, JobMacroGraph dependencyGraph, Map<String, Set<TextLocation>> macroSearches) {
String config = jobConfigManager.getConfig(job.getId());
IncludeLocations macroIncludeLocations = forMacros(config);
Predicate<String> predicate = pattern.asPredicate();
Set<TextLocation> searchLocs = LineSearch.search(config, pattern);
// For each macro dependency of the job, see if that macro (or any of its dependencies) contains a search result
searchLocs.addAll(getDependencySearchMatches(macroIncludeLocations, dependencyGraph, macroSearches));
// For each alias in the job, see if any of the job IDs which that alias point to contain a search result
// Macros and aliases have identical syntax -- the same locations map can be used for either one
searchLocs.addAll(getMatchedAliasLocations(macroIncludeLocations));
// For each parameter of the job, see if that parameter contains a search result
IncludeLocations paramIncludeLocations = IncludeLocations.forJobParams(config);
for (JobParameter param : job.getParameters()) {
Set<TextLocation> paramLocations = paramIncludeLocations.locationsFor(param.getName());
// Do not test default parameter value because it is already done when checking job config/macro
String paramValue = param.getValue();
if (!Strings.isNullOrEmpty(paramValue) && predicate.test(paramValue)) {
searchLocs.addAll(paramLocations);
}
// Sadly, these parameters might ALSO contain macros, aliases etc. so test that too (note we use the
// effectual parameter value here)
IncludeLocations nestedIncludeLocations = forMacros(param.getValueOrDefault());
if (!getMatchedAliasLocations(nestedIncludeLocations).isEmpty()) {
searchLocs.addAll(paramLocations);
}
if (!getDependencySearchMatches(nestedIncludeLocations, dependencyGraph, macroSearches).isEmpty()) {
searchLocs.addAll(paramLocations);
}
}
// Merge the matches together into groups which can be easily displayed on the client
List<AdjacentMatchesBlock> groups = AdjacentMatchesBlock.mergeMatchList(config.split("\n"), searchLocs);
if (groups.size() > 0) {
return new SearchResult(job.getId(), job.getDescription(), groups);
} else {
return null;
}
}
private Set<TextLocation> getMatchedAliasLocations(IncludeLocations macroIncludeLocations) {
Predicate<String> predicate = pattern.asPredicate();
ImmutableSet.Builder<TextLocation> results = ImmutableSet.builder();
for (String dep : macroIncludeLocations.dependencies()) {
// dep may be an alias or a macro - macroIncludeLocations may contain both because both are denoted using
// %{...}%. For an alias, check if any of its values match the search pattern.
List<String> jobIds = aliases.get(dep);
if (jobIds != null) {
for (String jobId : jobIds) {
if (predicate.test(jobId)) {
results.addAll(macroIncludeLocations.locationsFor(dep));
break;
}
}
}
}
return results.build();
}
@Nullable
private SearchResult getMacroSearchResult(String macroName, Set<TextLocation> macroSearch) {
JobMacro macro = macros.get(macroName);
if (macro == null) {
throw new NullPointerException();
}
String[] macroLines = macro.getMacro().split("\n");
List<AdjacentMatchesBlock> adjacentMatchesBlocks = AdjacentMatchesBlock.mergeMatchList(macroLines, macroSearch);
if (adjacentMatchesBlocks.size() > 0) {
return new SearchResult(macroName, "", adjacentMatchesBlocks);
} else {
return null;
}
}
private List<SearchResult> getMacroSearchResults(Map<String, Set<TextLocation>> macroSearches) {
List<SearchResult> results = new ArrayList<>();
for (String macroName : macroSearches.keySet()) {
SearchResult result = getMacroSearchResult(macroName, macroSearches.get(macroName));
if (result != null) {
results.add(result);
}
}
return results;
}
/**
* Finds all macros and their search match locations, if any.
* <p/>
* A macro may have 0 or more match locations. A match may be direct or indirect. A direct match location is where
* the search pattern is located in the macro. An indirect match location is where another macro is included that
* contains a direct or indirect match.
*
* @param macros all macros
* @param dependencyGraph provides macro dependencies
* @return A map of macro names to their match locations. If a macro has no match, its value will be an empty set.
*/
private Map<String, Set<TextLocation>> searchMacros(Map<String, JobMacro> macros, JobMacroGraph dependencyGraph) {
Map<String, Set<TextLocation>> results = new HashMap<>();
// Search the macro texts for direct match of the search pattern
for (String macroName : macros.keySet()) {
JobMacro macro = macros.get(macroName);
results.put(macroName, LineSearch.search(macro.getMacro(), pattern));
}
// Search the marco texts for job parameters whose assigned value on any job matches the search pattern
Map<String, Map<String, Set<TextLocation>>> paramMacroLocations = buildJobParameterMacroMap(macros);
Predicate<String> predicate = pattern.asPredicate();
for (Job job : jobs.values()) {
for (JobParameter param : job.getParameters()) {
// Test every job parameter value for match. For a matching parameter, add all macros that references it
// Do not test default parameter value because it will be included when checking job config/macro body
String paramValue = param.getValue();
if (!Strings.isNullOrEmpty(paramValue) && predicate.test(paramValue)) {
// all macros containing this parameter are potential matches
Map<String, Set<TextLocation>> potentialMacros = paramMacroLocations.get(param.getName());
if ((potentialMacros != null) && !potentialMacros.isEmpty()) {
Set<String> jobIncludedMacros = getJobIncludedMacros(job.getId(), dependencyGraph);
// filter potential macros down to this job's macros only
Map<String, Set<TextLocation>> matchingMacros = potentialMacros.entrySet().stream()
.filter(p -> jobIncludedMacros.contains(p.getKey()))
.collect(Collectors.toMap(p -> p.getKey(), p -> p.getValue()));
mergeToLocationsMap(matchingMacros, results);
}
}
}
}
// Macros can include other macros, so we need to add dependent macros (which contain search results) to the
// parent macro's search results.
for (String macroName : results.keySet()) {
Set<TextLocation> macroSearchResults = results.get(macroName);
IncludeLocations macroIncludeLocations = dependencyGraph.getIncludeLocations(macroName);
// See if any of the (recursive) dependencies of this macro had search results. If they do, we add a new
// LineMatch to this macro's search results, which indicates where the macro w/ a search result was included
macroSearchResults.addAll(getDependencySearchMatches(macroIncludeLocations, dependencyGraph, results));
}
return results;
}
/**
* Returns all macros included directly or indirectly in a job.
*
* @param jobId the job id
* @param dependencyGraph used to find indirectly included macros (i.e. macros included in another macro)
*/
private Set<String> getJobIncludedMacros(String jobId, JobMacroGraph dependencyGraph) {
Set<String> macros = jobMacrosMap.get(jobId);
if (macros == null) {
// get macros directly included in the job config
String jobConfig = jobConfigManager.getConfig(jobId);
Set<String> directMacros = IncludeLocations.forMacros(jobConfig).dependencies();
if (directMacros.isEmpty()) {
macros = Collections.emptySet();
} else {
// get all the indirectly included macros too
macros = directMacros.stream()
.flatMap(m -> dependencyGraph.getDependencies(m).stream())
.collect(Collectors.toSet());
}
jobMacrosMap.put(jobId, macros);
}
return macros;
}
private void mergeToLocationsMap(@Nullable Map<String, Set<TextLocation>> from, Map<String, Set<TextLocation>> to) {
if ((from == null) || from.isEmpty()) {
return;
}
for (Map.Entry<String, Set<TextLocation>> entry : from.entrySet()) {
String key = entry.getKey();
Set<TextLocation> toLocations = to.get(key);
if (toLocations == null) {
toLocations = new HashSet<>();
to.put(key, toLocations);
}
toLocations.addAll(entry.getValue());
}
}
/**
* Returns a map of parameter names to all the macros that include the parameter in its text.
*
* @param macros
* @return the key is parameter name, the value is a map of all macros containing the parameter and the
* locations where the parameter is included in each macro. Parameter to macro is one-to-many
* because muliple macros may include the same parameter; macro to location is one-to-many
* because a macro may include the same parameter in multiple places.
*/
private Map<String, Map<String, Set<TextLocation>>> buildJobParameterMacroMap(Map<String, JobMacro> macros) {
Map<String, Map<String, Set<TextLocation>>> result = new HashMap<>();
for (Map.Entry<String, JobMacro> entry : macros.entrySet()) {
String macroName = entry.getKey();
String macroBody = entry.getValue().getMacro();
IncludeLocations allParamLocations = IncludeLocations.forJobParams(macroBody);
for (String paramName : allParamLocations.dependencies()) {
Map<String, Set<TextLocation>> paramMacros = result.get(paramName);
if (paramMacros == null) {
paramMacros = new HashMap<>();
result.put(paramName, paramMacros);
}
Set<TextLocation> paramLocations = allParamLocations.locationsFor(paramName);
if (!paramLocations.isEmpty()) {
paramMacros.put(macroName, paramLocations);
}
}
}
return result;
}
/**
* Finds among a list of macros those that contain (recursively) a search match and returns their locations.
* <p/>
* A list of macros and the locations where they are included (in a job config or a macro) are provided to this
* method, along with the full macro dependency graph and the complete macro search match results. For each macro
* in the list, this method looks at the macro itself and all its direct and indirect dependencies; if any has a
* search match, the included macro's locations are added to the result set that will be returned.
*
* @param macroIncludeLocations the macros and their inclusion locations (in a job config or a macro)
* @param dependencyGraph the full macro dependency graph
* @param macroSearchResults all macros and their search match locations (if any)
* @return the inclusion locations from <code>macroIncludeLocations</code> for the macros that have a search match
* in itself or one of its dependencies (direct or indirect)
*/
private Set<TextLocation> getDependencySearchMatches(IncludeLocations macroIncludeLocations,
JobMacroGraph dependencyGraph,
Map<String, Set<TextLocation>> macroSearchResults) {
ImmutableSet.Builder<TextLocation> builder = ImmutableSet.builder();
for (String depMacroName : macroIncludeLocations.dependencies()) {
// For each dependency that depMacroName brings in, see if any of THEM have search results. If they do, we
// want to link back to the include to depMacroName in the search result.
for (String deeperDepName : dependencyGraph.getDependencies(depMacroName)) {
Set<TextLocation> depMacroResults = macroSearchResults.getOrDefault(deeperDepName, ImmutableSet.of());
if (!depMacroResults.isEmpty()) {
builder.addAll(macroIncludeLocations.locationsFor(depMacroName));
break;
}
}
}
return builder.build();
}
}