/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.ambari.logfeeder.common; import java.io.BufferedInputStream; import java.io.File; import java.lang.reflect.Type; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.ambari.logfeeder.filter.Filter; import org.apache.ambari.logfeeder.input.Input; import org.apache.ambari.logfeeder.input.InputManager; import org.apache.ambari.logfeeder.input.InputSimulate; import org.apache.ambari.logfeeder.metrics.MetricData; import org.apache.ambari.logfeeder.output.Output; import org.apache.ambari.logfeeder.output.OutputManager; import org.apache.ambari.logfeeder.util.AliasUtil; import org.apache.ambari.logfeeder.util.FileUtil; import org.apache.ambari.logfeeder.util.LogFeederUtil; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.ambari.logfeeder.util.AliasUtil.AliasType; import org.apache.ambari.logsearch.config.api.InputConfigMonitor; import org.apache.log4j.Level; import org.apache.log4j.Logger; import com.google.common.collect.ImmutableMap; import com.google.gson.reflect.TypeToken; public class ConfigHandler implements InputConfigMonitor { private static final Logger LOG = Logger.getLogger(ConfigHandler.class); private final OutputManager outputManager = new OutputManager(); private final InputManager inputManager = new InputManager(); public static Map<String, Object> globalConfigs = new HashMap<>(); private final List<Map<String, Object>> inputConfigList = new ArrayList<>(); private final List<Map<String, Object>> filterConfigList = new ArrayList<>(); private final List<Map<String, Object>> outputConfigList = new ArrayList<>(); private boolean simulateMode = false; public ConfigHandler() {} public void init() throws Exception { loadConfigFiles(); loadOutputs(); simulateIfNeeded(); inputManager.init(); outputManager.init(); } private void loadConfigFiles() throws Exception { List<String> configFiles = getConfigFiles(); for (String configFileName : configFiles) { LOG.info("Going to load config file:" + configFileName); configFileName = configFileName.replace("\\ ", "%20"); File configFile = new File(configFileName); if (configFile.exists() && configFile.isFile()) { LOG.info("Config file exists in path." + configFile.getAbsolutePath()); loadConfigsUsingFile(configFile); } else { LOG.info("Trying to load config file from classloader: " + configFileName); loadConfigsUsingClassLoader(configFileName); LOG.info("Loaded config file from classloader: " + configFileName); } } } private List<String> getConfigFiles() { List<String> configFiles = new ArrayList<>(); String logfeederConfigFilesProperty = LogFeederUtil.getStringProperty("logfeeder.config.files"); LOG.info("logfeeder.config.files=" + logfeederConfigFilesProperty); if (logfeederConfigFilesProperty != null) { configFiles.addAll(Arrays.asList(logfeederConfigFilesProperty.split(","))); } String inputConfigDir = LogFeederUtil.getStringProperty("input_config_dir"); if (StringUtils.isNotEmpty(inputConfigDir)) { File configDirFile = new File(inputConfigDir); List<File> inputConfigFiles = FileUtil.getAllFileFromDir(configDirFile, "json", false); for (File inputConfigFile : inputConfigFiles) { configFiles.add(inputConfigFile.getAbsolutePath()); } } if (CollectionUtils.isEmpty(configFiles)) { String configFileProperty = LogFeederUtil.getStringProperty("config.file", "config.json"); configFiles.addAll(Arrays.asList(configFileProperty.split(","))); } return configFiles; } private void loadConfigsUsingFile(File configFile) throws Exception { try { String configData = FileUtils.readFileToString(configFile, Charset.defaultCharset()); loadConfigs(configData); } catch (Exception t) { LOG.error("Error opening config file. configFilePath=" + configFile.getAbsolutePath()); throw t; } } private void loadConfigsUsingClassLoader(String configFileName) throws Exception { try (BufferedInputStream fis = (BufferedInputStream) this.getClass().getClassLoader().getResourceAsStream(configFileName)) { String configData = IOUtils.toString(fis, Charset.defaultCharset()); loadConfigs(configData); } } @Override public void loadInputConfigs(String serviceName, String inputConfigData) throws Exception { inputConfigList.clear(); filterConfigList.clear(); loadConfigs(inputConfigData); if (simulateMode) { InputSimulate.loadTypeToFilePath(inputConfigList); } else { loadInputs(serviceName); loadFilters(serviceName); assignOutputsToInputs(serviceName); inputManager.startInputs(serviceName); } } @Override public void removeInputs(String serviceName) { inputManager.removeInputsForService(serviceName); } @SuppressWarnings("unchecked") public void loadConfigs(String configData) throws Exception { Type type = new TypeToken<Map<String, Object>>() {}.getType(); Map<String, Object> configMap = LogFeederUtil.getGson().fromJson(configData, type); // Get the globals for (String key : configMap.keySet()) { switch (key) { case "global" : globalConfigs.putAll((Map<String, Object>) configMap.get(key)); break; case "input" : List<Map<String, Object>> inputConfig = (List<Map<String, Object>>) configMap.get(key); inputConfigList.addAll(inputConfig); break; case "filter" : List<Map<String, Object>> filterConfig = (List<Map<String, Object>>) configMap.get(key); filterConfigList.addAll(filterConfig); break; case "output" : List<Map<String, Object>> outputConfig = (List<Map<String, Object>>) configMap.get(key); outputConfigList.addAll(outputConfig); break; default : LOG.warn("Unknown config key: " + key); } } } private void simulateIfNeeded() throws Exception { int simulatedInputNumber = LogFeederUtil.getIntProperty("logfeeder.simulate.input_number", 0); if (simulatedInputNumber == 0) return; List<Map<String, Object>> simulateInputConfigList = new ArrayList<>(); for (int i = 0; i < simulatedInputNumber; i++) { HashMap<String, Object> mapList = new HashMap<String, Object>(); mapList.put("source", "simulate"); mapList.put("rowtype", "service"); simulateInputConfigList.add(mapList); } Map<String, List<Map<String, Object>>> simulateInputConfigMap = ImmutableMap.of("input", simulateInputConfigList); String simulateInputConfig = LogFeederUtil.getGson().toJson(simulateInputConfigMap); loadInputConfigs("Simulation", simulateInputConfig); simulateMode = true; } private void loadOutputs() { for (Map<String, Object> map : outputConfigList) { if (map == null) { continue; } mergeBlocks(globalConfigs, map); String value = (String) map.get("destination"); if (StringUtils.isEmpty(value)) { LOG.error("Output block doesn't have destination element"); continue; } Output output = (Output) AliasUtil.getClassInstance(value, AliasType.OUTPUT); if (output == null) { LOG.error("Output object could not be found"); continue; } output.setDestination(value); output.loadConfig(map); // We will only check for is_enabled out here. Down below we will check whether this output is enabled for the input if (output.getBooleanValue("is_enabled", true)) { output.logConfigs(Level.INFO); outputManager.add(output); } else { LOG.info("Output is disabled. So ignoring it. " + output.getShortDescription()); } } } private void loadInputs(String serviceName) { for (Map<String, Object> map : inputConfigList) { if (map == null) { continue; } mergeBlocks(globalConfigs, map); String value = (String) map.get("source"); if (StringUtils.isEmpty(value)) { LOG.error("Input block doesn't have source element"); continue; } Input input = (Input) AliasUtil.getClassInstance(value, AliasType.INPUT); if (input == null) { LOG.error("Input object could not be found"); continue; } input.setType(value); input.loadConfig(map); if (input.isEnabled()) { input.setOutputManager(outputManager); input.setInputManager(inputManager); inputManager.add(serviceName, input); input.logConfigs(Level.INFO); } else { LOG.info("Input is disabled. So ignoring it. " + input.getShortDescription()); } } } private void loadFilters(String serviceName) { sortFilters(); List<Input> toRemoveInputList = new ArrayList<Input>(); for (Input input : inputManager.getInputList(serviceName)) { for (Map<String, Object> map : filterConfigList) { if (map == null) { continue; } mergeBlocks(globalConfigs, map); String value = (String) map.get("filter"); if (StringUtils.isEmpty(value)) { LOG.error("Filter block doesn't have filter element"); continue; } Filter filter = (Filter) AliasUtil.getClassInstance(value, AliasType.FILTER); if (filter == null) { LOG.error("Filter object could not be found"); continue; } filter.loadConfig(map); filter.setInput(input); if (filter.isEnabled()) { filter.setOutputManager(outputManager); input.addFilter(filter); filter.logConfigs(Level.INFO); } else { LOG.debug("Ignoring filter " + filter.getShortDescription() + " for input " + input.getShortDescription()); } } if (input.getFirstFilter() == null) { toRemoveInputList.add(input); } } for (Input toRemoveInput : toRemoveInputList) { LOG.warn("There are no filters, we will ignore this input. " + toRemoveInput.getShortDescription()); inputManager.removeInput(toRemoveInput); } } private void sortFilters() { Collections.sort(filterConfigList, new Comparator<Map<String, Object>>() { @Override public int compare(Map<String, Object> o1, Map<String, Object> o2) { Object o1Sort = o1.get("sort_order"); Object o2Sort = o2.get("sort_order"); if (o1Sort == null || o2Sort == null) { return 0; } int o1Value = parseSort(o1, o1Sort); int o2Value = parseSort(o2, o2Sort); return o1Value - o2Value; } private int parseSort(Map<String, Object> map, Object o) { if (!(o instanceof Number)) { try { return (new Double(Double.parseDouble(o.toString()))).intValue(); } catch (Throwable t) { LOG.error("Value is not of type Number. class=" + o.getClass().getName() + ", value=" + o.toString() + ", map=" + map.toString()); return 0; } } else { return ((Number) o).intValue(); } } }); } private void assignOutputsToInputs(String serviceName) { Set<Output> usedOutputSet = new HashSet<Output>(); for (Input input : inputManager.getInputList(serviceName)) { for (Output output : outputManager.getOutputs()) { if (LogFeederUtil.isEnabled(output.getConfigs(), input.getConfigs())) { usedOutputSet.add(output); input.addOutput(output); } } } // In case of simulation copies of the output are added for each simulation instance, these must be added to the manager for (Output output : InputSimulate.getSimulateOutputs()) { outputManager.add(output); usedOutputSet.add(output); } } @SuppressWarnings("unchecked") private void mergeBlocks(Map<String, Object> fromMap, Map<String, Object> toMap) { for (String key : fromMap.keySet()) { Object objValue = fromMap.get(key); if (objValue == null) { continue; } if (objValue instanceof Map) { Map<String, Object> globalFields = LogFeederUtil.cloneObject((Map<String, Object>) objValue); Map<String, Object> localFields = (Map<String, Object>) toMap.get(key); if (localFields == null) { localFields = new HashMap<String, Object>(); toMap.put(key, localFields); } if (globalFields != null) { for (String fieldKey : globalFields.keySet()) { if (!localFields.containsKey(fieldKey)) { localFields.put(fieldKey, globalFields.get(fieldKey)); } } } } } // Let's add the rest of the top level fields if missing for (String key : fromMap.keySet()) { if (!toMap.containsKey(key)) { toMap.put(key, fromMap.get(key)); } } } public void cleanCheckPointFiles() { inputManager.cleanCheckPointFiles(); } public void logStats() { inputManager.logStats(); outputManager.logStats(); } public void addMetrics(List<MetricData> metricsList) { inputManager.addMetricsContainers(metricsList); outputManager.addMetricsContainers(metricsList); } public void waitOnAllInputs() { inputManager.waitOnAllInputs(); } public void close() { inputManager.close(); outputManager.close(); inputManager.checkInAll(); } }