ExtensionStore.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.falcon.extensions.store;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.entity.parser.ValidationException;
import org.apache.falcon.entity.store.StoreAccessException;
import org.apache.falcon.extensions.AbstractExtension;
import org.apache.falcon.extensions.ExtensionStatus;
import org.apache.falcon.extensions.ExtensionType;
import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.falcon.persistence.ExtensionBean;
import org.apache.falcon.util.StartupProperties;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Store for Falcon extensions.
 */
public final class ExtensionStore {

    private static final Logger LOG = LoggerFactory.getLogger(ExtensionStore.class);

    public static ExtensionMetaStore getMetaStore() {
        return metaStore;
    }

    private static ExtensionMetaStore metaStore = new ExtensionMetaStore();
    private FileSystem fs;

    private Path storePath;

    private static final String EXTENSION_PROPERTY_JSON_SUFFIX = "-properties.json";
    private static final String SHORT_DESCRIPTION = "shortDescription";

    // Convention over configuration design paradigm
    private static final String RESOURCES_DIR = "resources";
    private static final String LIBS_DIR = "libs";

    static final String EXTENSION_STORE_URI = "extension.store.uri";

    private static final ExtensionStore STORE = new ExtensionStore();


    public static ExtensionStore get() {
        return STORE;
    }

    private ExtensionStore() {
        String uri = StartupProperties.get().getProperty(EXTENSION_STORE_URI);
        if (StringUtils.isEmpty(uri)) {
            throw new RuntimeException("Property extension.store.uri not set in startup properties."
                    + "Please set it to path of extension deployment on HDFS. Extension store init failed");
        }
        storePath = new Path(uri);
        fs = initializeFileSystem();
        initializeDbTable();
    }

    private void initializeDbTable() {
        try {
            metaStore.deleteExtensionsOfType(ExtensionType.TRUSTED);
            List<String> extensions = getTrustedExtensions();
            for (String extension : extensions) {
                ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extension)
                        ? ExtensionType.TRUSTED : ExtensionType.CUSTOM;
                String description = getShortDescription(extension);
                String location = storePath.toString() + '/' + extension;
                String extensionOwner = System.getProperty("user.name");
                metaStore.storeExtensionBean(extension, location, extensionType, description, extensionOwner);
            }
        } catch (FalconException e) {
            LOG.error("Exception in ExtensionMetaStore:", e);
            throw new RuntimeException(e);
        }

    }

    private String getShortDescription(final String extensionName) throws FalconException {
        String location = storePath.toString() + "/" + extensionName + "/META/"
                + extensionName.toLowerCase() + EXTENSION_PROPERTY_JSON_SUFFIX;
        String content = getResource(location);
        String description;
        try {
            JSONObject jsonObject = new JSONObject(content);
            description = (String) jsonObject.get(SHORT_DESCRIPTION);
        } catch (JSONException e) {
            throw new FalconException(e);
        }
        return description;
    }

    private FileSystem initializeFileSystem() {
        try {
            FileSystem fileSystem =
                    HadoopClientFactory.get().createFalconFileSystem(storePath.toUri());
            if (!fileSystem.exists(storePath)) {
                LOG.info("Creating extension store directory: {}", storePath);
                // set permissions so config store dir is owned by falcon alone
                HadoopClientFactory.mkdirs(fileSystem, storePath, HadoopClientFactory.ALL_PERMISSION);
            }
            return fileSystem;
        } catch (Exception e) {
            throw new RuntimeException("Unable to bring up extension store for path: " + storePath, e);
        }
    }


    public Map<String, String> getExtensionResources(final String extensionName) throws StoreAccessException {
        Map<String, String> extensionFileMap = new HashMap<>();
        try {
            Path extensionPath = new Path(storePath, extensionName.toLowerCase());

            Path resourcesPath = null;
            FileStatus[] files = fs.listStatus(extensionPath);

            for (FileStatus fileStatus : files) {
                if (fileStatus.getPath().getName().equalsIgnoreCase(RESOURCES_DIR)) {
                    resourcesPath = fileStatus.getPath();
                    break;
                }
            }

            if (resourcesPath == null) {
                throw new StoreAccessException(" For extension " + extensionName
                        + " there is no " + RESOURCES_DIR + "at the extension store path " + storePath);
            }
            RemoteIterator<LocatedFileStatus> fileStatusListIterator = fs.listFiles(resourcesPath, true);
            while (fileStatusListIterator.hasNext()) {
                LocatedFileStatus fileStatus = fileStatusListIterator.next();
                Path filePath = Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath());
                extensionFileMap.put(filePath.getName(), filePath.toString());
            }
        } catch (IOException e) {
            throw new StoreAccessException(e);
        }
        return extensionFileMap;
    }

    public String getExtensionLibPath(final String extensionName) throws StoreAccessException {
        try {
            Path extensionPath = new Path(storePath, extensionName.toLowerCase());

            Path libsPath = null;
            FileStatus[] files = fs.listStatus(extensionPath);

            for (FileStatus fileStatus : files) {
                if (fileStatus.getPath().getName().equalsIgnoreCase(LIBS_DIR)) {
                    libsPath = Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath());
                    break;
                }
            }

            if (libsPath == null) {
                LOG.info("For extension " + extensionName + " there is no "
                        + LIBS_DIR + "at the extension store path " + extensionPath);
                return null;
            } else {
                return libsPath.toString();
            }
        } catch (IOException e) {
            throw new StoreAccessException(e);
        }
    }

    public String getExtensionResource(final String resourcePath) throws FalconException {
        if (StringUtils.isBlank(resourcePath)) {
            throw new StoreAccessException("Resource path cannot be null or empty");
        }

        try {
            Path resourceFile = new Path(resourcePath);
            InputStream data;

            ByteArrayOutputStream writer = new ByteArrayOutputStream();
            if (resourcePath.startsWith("file")) {
                data = fs.open(resourceFile);
                IOUtils.copyBytes(data, writer, fs.getConf(), true);
            } else {
                FileSystem fileSystem = getHdfsFileSystem(resourcePath);
                data = fileSystem.open(resourceFile);
                IOUtils.copyBytes(data, writer, fileSystem.getConf(), true);
            }
            return writer.toString();
        } catch (IOException e) {
            throw new StoreAccessException(e);
        }
    }

    private List<String> getTrustedExtensions() throws StoreAccessException {
        List<String> extensionList = new ArrayList<>();
        try {
            FileStatus[] fileStatuses = fs.listStatus(storePath);

            for (FileStatus fileStatus : fileStatuses) {
                if (fileStatus.isDirectory()) {
                    Path filePath = Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath());
                    extensionList.add(filePath.getName());
                }
            }
        } catch (IOException e) {
            throw new StoreAccessException(e);
        }
        return extensionList;
    }

    public String deleteExtension(final String extensionName, String currentUser) throws FalconException {
        ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extensionName) ? ExtensionType.TRUSTED
                : ExtensionType.CUSTOM;
        if (extensionType.equals(ExtensionType.TRUSTED)) {
            throw new ValidationException(extensionName + " is trusted cannot be deleted.");
        } else if (!metaStore.checkIfExtensionExists(extensionName)) {
            throw new FalconException("Extension:" + extensionName + " is not registered with Falcon.");
        } else if (!metaStore.getDetail(extensionName).getExtensionOwner().equals(currentUser)) {
            throw new FalconException("User: " + currentUser + " is not allowed to delete extension: " + extensionName);
        } else {
            metaStore.deleteExtension(extensionName);
            return "Deleted extension:" + extensionName;
        }
    }

    private void assertURI(String part, String value) throws ValidationException {
        if (value == null) {
            String msg = "Invalid Path supplied. " + part + " is missing. "
                    + " Path must contain scheme, authority and path.";
            LOG.error(msg);
            throw new ValidationException(msg);
        }
    }

    private FileSystem getHdfsFileSystem(String path)  throws  FalconException {
        URI uri;
        try {
            uri = new URI(path);
        } catch (URISyntaxException e) {
            LOG.error("Exception : ", e);
            throw new FalconException(e);
        }
        return HadoopClientFactory.get().createFalconFileSystem(uri);
    }


    public String registerExtension(final String extensionName, final String path, final String description,
                                    String extensionOwner) throws URISyntaxException, FalconException {
        if (!metaStore.checkIfExtensionExists(extensionName)) {
            URI uri = new URI(path);
            assertURI("Scheme", uri.getScheme());
            assertURI("Authority", uri.getAuthority());
            assertURI("Path", uri.getPath());
            FileSystem fileSystem = getHdfsFileSystem(path);
            try {
                fileSystem.listStatus(new Path(uri.getPath() + "/README"));
            } catch (IOException e) {
                LOG.error("Exception in registering Extension:{}", extensionName, e);
                throw new ValidationException("README file is not present in the " + path);
            }
            PathFilter filter = new PathFilter() {
                public boolean accept(Path file) {
                    return file.getName().endsWith(".jar");
                }
            };
            FileStatus[] jarStatus;
            try {
                jarStatus = fileSystem.listStatus(new Path(uri.getPath(), "libs/build"), filter);
                if (jarStatus.length <= 0) {
                    throw new ValidationException("Jars are not present in the " + uri.getPath() + "/libs/build.");
                }
            } catch (IOException e) {
                LOG.error("Exception in registering Extension:{}", extensionName, e);
                throw new ValidationException("Jars are not present in the " + uri.getPath() + "/libs/build.");
            }

            FileStatus[] propStatus;
            try {
                propStatus = fileSystem.listStatus(new Path(uri.getPath() , "META"));
                if (propStatus.length <= 0) {
                    throw new ValidationException("No properties file is not present in the " + uri.getPath() + "/META"
                            + " structure.");
                }
            } catch (IOException e) {
                LOG.error("Exception in registering Extension:{}", extensionName, e);
                throw new ValidationException("Directory is not present in the " + uri.getPath() + "/META"
                        + " structure.");
            }
            metaStore.storeExtensionBean(extensionName, path, ExtensionType.CUSTOM, description, extensionOwner);
        } else {
            throw new ValidationException(extensionName + " already exists.");
        }
        LOG.info("Extension :" + extensionName + " registered successfully.");
        return "Extension :" + extensionName + " registered successfully.";
    }

    public String getResource(final String extensionResourcePath)
        throws FalconException {
        StringBuilder definition = new StringBuilder();
        Path resourcePath = new Path(extensionResourcePath);
        FileSystem fileSystem = HadoopClientFactory.get().createFalconFileSystem(resourcePath.toUri());
        try {
            if (fileSystem.isFile(resourcePath)) {
                definition.append(getExtensionResource(extensionResourcePath.toString()));
            } else {
                RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem.listFiles(resourcePath, false);
                while (fileStatusListIterator.hasNext()) {
                    LocatedFileStatus fileStatus = fileStatusListIterator.next();
                    Path filePath = fileStatus.getPath();
                    definition.append("Contents of file ").append(filePath.getName()).append(":\n");
                    definition.append(getExtensionResource(filePath.toString())).append("\n \n");
                }
            }
        } catch (IOException e) {
            LOG.error("Exception while getting file(s) with path : " + extensionResourcePath, e);
            throw new StoreAccessException(e);
        }

        return definition.toString();

    }

    public Path getExtensionStorePath() {
        return storePath;
    }

    public boolean isExtensionStoreInitialized() {
        return (storePath != null);
    }

    public String updateExtensionStatus(final String extensionName, String currentUser, ExtensionStatus status) throws
            FalconException {
        validateStatusChange(extensionName, currentUser);
        ExtensionBean extensionBean = metaStore.getDetail(extensionName);
        if (extensionBean == null) {
            LOG.error("Extension not found: " + extensionName);
            throw new FalconException("Extension not found:" + extensionName);
        }
        if (extensionBean.getStatus().equals(status)) {
            throw new ValidationException(extensionName + " is already in " + status.toString() + " state.");
        } else {
            metaStore.updateExtensionStatus(extensionName, status);
            return "Status of extension: " + extensionName + "changed to " + status.toString() + " state.";
        }
    }

    private void validateStatusChange(final String extensionName, String currentUser) throws FalconException {

        ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extensionName) ? ExtensionType.TRUSTED
                : ExtensionType.CUSTOM;
        if (extensionType.equals(ExtensionType.TRUSTED)) {
            throw new ValidationException(extensionName + " is trusted. Status can't be changed for trusted "
                    + "extensions.");
        } else if (!metaStore.checkIfExtensionExists(extensionName)) {
            throw new FalconException("Extension:" + extensionName + " is not registered with Falcon.");
        } else if (!metaStore.getDetail(extensionName).getExtensionOwner().equals(currentUser)) {
            throw new FalconException("User: " + currentUser + " is not allowed to change status of extension: "
                    + extensionName);
        }
    }

}