package azkaban.project.validator;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.log4j.Logger;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import azkaban.project.Project;
import azkaban.project.DirectoryFlowLoader;
import azkaban.utils.Props;
/**
* Xml implementation of the ValidatorManager. Looks for the property
* project.validators.xml.file in the azkaban properties.
*
* The xml to be in the following form:
* <azkaban-validators>
* <validator classname="validator class name">
* <!-- optional configurations for each individual validator -->
* <property key="validator property key" value="validator property value" />
* ...
* </validator>
* </azkaban-validators>
*/
public class XmlValidatorManager implements ValidatorManager {
private static final Logger logger = Logger.getLogger(XmlValidatorManager.class);
public static final String AZKABAN_VALIDATOR_TAG = "azkaban-validators";
public static final String VALIDATOR_TAG = "validator";
public static final String CLASSNAME_ATTR = "classname";
public static final String ITEM_TAG = "property";
public static final String DEFAULT_VALIDATOR_KEY = "Directory Flow";
private static Map<String, Long> resourceTimestamps = new HashMap<String, Long>();
private static ValidatorClassLoader validatorLoader;
private Map<String, ProjectValidator> validators;
private String validatorDirPath;
/**
* Load the validator plugins from the validator directory (default being validators/) into
* the validator ClassLoader. This enables creating instances of these validators in the
* loadValidators() method.
*
* @param props
*/
public XmlValidatorManager(Props props) {
validatorDirPath = props.getString(ValidatorConfigs.VALIDATOR_PLUGIN_DIR, ValidatorConfigs.DEFAULT_VALIDATOR_DIR);
File validatorDir = new File(validatorDirPath);
if (!validatorDir.canRead() || !validatorDir.isDirectory()) {
logger.warn("Validator directory " + validatorDirPath
+ " does not exist or is not a directory.");
}
// Check for updated validator JAR files
checkResources();
// Load the validators specified in the xml file.
try {
loadValidators(props, logger);
} catch (Exception e) {
logger.error("Cannot load all the validators.");
throw new ValidatorManagerException(e);
}
}
private void checkResources() {
File validatorDir = new File(validatorDirPath);
List<URL> resources = new ArrayList<URL>();
boolean reloadResources = false;
try {
if (validatorDir.canRead() && validatorDir.isDirectory()) {
for (File f : validatorDir.listFiles()) {
if (f.getName().endsWith(".jar")) {
resources.add(f.toURI().toURL());
if (resourceTimestamps.get(f.getName()) == null
|| resourceTimestamps.get(f.getName()) != f.lastModified()) {
reloadResources = true;
logger.info("Resource " + f.getName() + " is updated. Reload the classloader.");
resourceTimestamps.put(f.getName(), f.lastModified());
}
}
}
}
} catch (MalformedURLException e) {
throw new ValidatorManagerException(e);
}
if (reloadResources) {
if (validatorLoader != null) {
try {
// Since we cannot use Java 7 feature inside Azkaban (....), we need a customized class loader
// that does the close for us.
validatorLoader.close();
} catch (ValidatorManagerException e) {
logger.error("Cannot reload validator classloader because failure "
+ "to close the validator classloader.", e);
// We do not throw the ValidatorManagerException because we do not want to crash Azkaban at runtime.
}
}
validatorLoader = new ValidatorClassLoader(resources.toArray(new URL[resources.size()]));
}
}
/**
* Instances of the validators are created here rather than in the constructors. This is because
* some validators might need to maintain project-specific states, such as {@link DirectoryFlowLoader}.
* By instantiating the validators here, it ensures that the validator objects are project-specific,
* rather than global.
*
* {@inheritDoc}
* @see azkaban.project.validator.ValidatorManager#loadValidators(azkaban.utils.Props, org.apache.log4j.Logger)
*/
@Override
public void loadValidators(Props props, Logger log) {
validators = new LinkedHashMap<String, ProjectValidator>();
// Add the default validator
DirectoryFlowLoader flowLoader = new DirectoryFlowLoader(props, log);
validators.put(flowLoader.getValidatorName(), flowLoader);
if (!props.containsKey(ValidatorConfigs.XML_FILE_PARAM)) {
logger.warn("Azkaban properties file does not contain the key " + ValidatorConfigs.XML_FILE_PARAM);
return;
}
String xmlPath = props.get(ValidatorConfigs.XML_FILE_PARAM);
File file = new File(xmlPath);
if (!file.exists()) {
logger.error("Azkaban validator configuration file " + xmlPath + " does not exist.");
return;
}
// Creating the document builder to parse xml.
DocumentBuilderFactory docBuilderFactory =
DocumentBuilderFactory.newInstance();
DocumentBuilder builder = null;
try {
builder = docBuilderFactory.newDocumentBuilder();
} catch (ParserConfigurationException e) {
throw new ValidatorManagerException(
"Exception while parsing validator xml. Document builder not created.", e);
}
Document doc = null;
try {
doc = builder.parse(file);
} catch (SAXException e) {
throw new ValidatorManagerException("Exception while parsing " + xmlPath
+ ". Invalid XML.", e);
} catch (IOException e) {
throw new ValidatorManagerException("Exception while parsing " + xmlPath
+ ". Error reading file.", e);
}
NodeList tagList = doc.getChildNodes();
Node azkabanValidators = tagList.item(0);
NodeList azkabanValidatorsList = azkabanValidators.getChildNodes();
for (int i = 0; i < azkabanValidatorsList.getLength(); ++i) {
Node node = azkabanValidatorsList.item(i);
if (node.getNodeType() == Node.ELEMENT_NODE) {
if (node.getNodeName().equals(VALIDATOR_TAG)) {
parseValidatorTag(node, props, log);
}
}
}
}
@SuppressWarnings("unchecked")
private void parseValidatorTag(Node node, Props props, Logger log) {
NamedNodeMap validatorAttrMap = node.getAttributes();
Node classNameAttr = validatorAttrMap.getNamedItem(CLASSNAME_ATTR);
if (classNameAttr == null) {
throw new ValidatorManagerException(
"Error loading validator. The validator 'classname' attribute doesn't exist");
}
NodeList keyValueItemsList = node.getChildNodes();
for (int i = 0; i < keyValueItemsList.getLength(); i++) {
Node keyValuePair = keyValueItemsList.item(i);
if (keyValuePair.getNodeName().equals(ITEM_TAG)) {
parseItemTag(keyValuePair, props);
}
}
String className = classNameAttr.getNodeValue();
try {
Class<? extends ProjectValidator> validatorClass =
(Class<? extends ProjectValidator>)validatorLoader.loadClass(className);
Constructor<?> validatorConstructor =
validatorClass.getConstructor(Logger.class);
ProjectValidator validator = (ProjectValidator) validatorConstructor.newInstance(log);
validator.initialize(props);
validators.put(validator.getValidatorName(), validator);
logger.info("Added validator " + className + " to list of validators.");
} catch (Exception e) {
logger.error("Could not instantiate ProjectValidator " + className);
throw new ValidatorManagerException(e);
}
}
private void parseItemTag(Node node, Props props) {
NamedNodeMap keyValueMap = node.getAttributes();
Node keyAttr = keyValueMap.getNamedItem("key");
Node valueAttr = keyValueMap.getNamedItem("value");
if (keyAttr == null || valueAttr == null) {
throw new ValidatorManagerException("Error loading validator key/value "
+ "pair. The 'key' or 'value' attribute doesn't exist");
}
props.put(keyAttr.getNodeValue(), valueAttr.getNodeValue());
}
@Override
public Map<String, ValidationReport> validate(Project project, File projectDir) {
Map<String, ValidationReport> reports = new LinkedHashMap<String, ValidationReport>();
for (Entry<String, ProjectValidator> validator : validators.entrySet()) {
reports.put(validator.getKey(), validator.getValue().validateProject(project, projectDir));
logger.info("Validation status of validator " + validator.getKey() + " is "
+ reports.get(validator.getKey()).getStatus());
}
return reports;
}
@Override
public ProjectValidator getDefaultValidator() {
return validators.get(DEFAULT_VALIDATOR_KEY);
}
@Override
public List<String> getValidatorsInfo() {
List<String> info = new ArrayList<String>();
for (String key : validators.keySet()) {
info.add(key);
}
return info;
}
}