OozieBundleBuilder.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.falcon.oozie;

import org.apache.commons.lang3.StringUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.entity.ClusterHelper;
import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.HiveUtil;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.falcon.oozie.bundle.BUNDLEAPP;
import org.apache.falcon.oozie.bundle.CONFIGURATION;
import org.apache.falcon.oozie.bundle.CONFIGURATION.Property;
import org.apache.falcon.oozie.bundle.COORDINATOR;
import org.apache.falcon.security.CurrentUser;
import org.apache.falcon.util.OozieUtils;
import org.apache.falcon.workflow.engine.AbstractWorkflowEngine;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.client.OozieClient;
import org.apache.falcon.Tag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.transform.stream.StreamSource;
import java.io.IOException;
import java.util.List;
import java.util.Map.Entry;
import java.util.Properties;

/**
 * Base class for building oozie bundle - bundle is the entity that falcon tracks in oozie.
 * @param <T>
 */
public abstract class OozieBundleBuilder<T extends Entity> extends OozieEntityBuilder<T> {
    public static final Logger LOG = LoggerFactory.getLogger(OozieBundleBuilder.class);

    public OozieBundleBuilder(T entity) {
        super(entity);
    }

    @Override
    public Properties build(Cluster cluster, Path buildPath) throws FalconException {
        String clusterName = cluster.getName();
        if (EntityUtil.getStartTime(entity, clusterName).compareTo(EntityUtil.getEndTime(entity, clusterName)) >= 0) {
            LOG.info("process validity start <= end for cluster {}. Skipping schedule", clusterName);
            return null;
        }

        List<Properties> coords = buildCoords(cluster, buildPath);
        if (coords == null || coords.isEmpty()) {
            return null;
        }

        BUNDLEAPP bundle = new BUNDLEAPP();
        bundle.setName(EntityUtil.getWorkflowName(entity).toString());
        // all the properties are set prior to bundle and coordinators creation

        for (Properties coordProps : coords) {
            // add the coordinator to the bundle
            COORDINATOR coord = new COORDINATOR();
            String coordPath = coordProps.getProperty(OozieEntityBuilder.ENTITY_PATH);
            final String coordName = coordProps.getProperty(OozieEntityBuilder.ENTITY_NAME);
            coord.setName(coordName);
            coord.setAppPath(getStoragePath(coordPath));
            coordProps.put(OozieClient.USER_NAME, CurrentUser.getUser());
            coordProps.setProperty(AbstractWorkflowEngine.NAME_NODE, ClusterHelper.getStorageUrl(cluster));
            if (EntityUtil.isTableStorageType(cluster, entity)) {
                Tag tag = EntityUtil.getWorkflowNameTag(coordName, entity);
                if (tag == Tag.REPLICATION) {
                    // todo: kludge send source hcat creds for coord dependency check to pass
                    String srcClusterName = EntityUtil.getWorkflowNameSuffix(coordName, entity);
                    coordProps.putAll(HiveUtil.getHiveCredentials(ClusterHelper.getCluster(srcClusterName)));
                } else {
                    coordProps.putAll(HiveUtil.getHiveCredentials(cluster));
                }
            }

            coord.setConfiguration(getConfig(coordProps));
            bundle.getCoordinator().add(coord);
        }

        Path marshalPath = marshal(cluster, bundle, buildPath); // write the bundle
        Properties properties = getProperties(marshalPath, entity.getName());
        properties.setProperty(OozieClient.BUNDLE_APP_PATH, getStoragePath(buildPath));
        properties.setProperty(AbstractWorkflowEngine.NAME_NODE, ClusterHelper.getStorageUrl(cluster));

        //Add libpath
        String libPath = getOozieLibPath(buildPath);
        if (StringUtils.isNotBlank(libPath)) {
            properties.put(OozieClient.LIBPATH, libPath);
        }

        return properties;
    }

    private String getOozieLibPath(final Path buildPath) {
        String path = getLibPath(buildPath);
        if (StringUtils.isBlank(path)) {
            return null;
        }
        StringBuilder storageLibPaths = new StringBuilder();
        String[] libPaths = path.split(EntityUtil.WF_LIB_SEPARATOR);
        for (String libPath : libPaths) {
            String tempPath = getStoragePath(libPath);
            if (StringUtils.isNotBlank(tempPath)) {
                if (StringUtils.isNotBlank(storageLibPaths)) {
                    storageLibPaths.append(EntityUtil.WF_LIB_SEPARATOR);
                }
                storageLibPaths.append(tempPath);
            }
        }
        return StringUtils.isBlank(storageLibPaths) ? null : storageLibPaths.toString();
    }

    protected CONFIGURATION getConfig(Properties props) {
        CONFIGURATION conf = new CONFIGURATION();
        for (Entry<Object, Object> prop : props.entrySet()) {
            Property confProp = new Property();
            confProp.setName((String) prop.getKey());
            confProp.setValue((String) prop.getValue());
            conf.getProperty().add(confProp);
        }
        return conf;
    }

    protected Path marshal(Cluster cluster, BUNDLEAPP bundle, Path outPath) throws FalconException {
        return marshal(cluster, new org.apache.falcon.oozie.bundle.ObjectFactory().createBundleApp(bundle),
                OozieUtils.BUNDLE_JAXB_CONTEXT, new Path(outPath, "bundle.xml"));
    }

    //Used by coordinator builders to return multiple coords
    //TODO Can avoid separate interface that returns list by building at lifecycle level
    protected abstract List<Properties> buildCoords(Cluster cluster, Path bundlePath) throws FalconException;

    public static BUNDLEAPP unmarshal(Cluster cluster, Path path) throws FalconException {
        try {
            FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(
                    path.toUri(), ClusterHelper.getConfiguration(cluster));
            Unmarshaller unmarshaller = OozieUtils.BUNDLE_JAXB_CONTEXT.createUnmarshaller();
            @SuppressWarnings("unchecked") JAXBElement<BUNDLEAPP> jaxbElement =
                    unmarshaller.unmarshal(new StreamSource(fs.open(path)), BUNDLEAPP.class);
            return jaxbElement.getValue();
        } catch (JAXBException e) {
            throw new FalconException(e);
        } catch (IOException e) {
            throw new FalconException(e);
        }
    }

    public abstract String getLibPath(Path buildPath);
}