OperationsMetacardSupport.java

/**
 * Copyright (c) Codice Foundation
 * <p/>
 * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser
 * General Public License as published by the Free Software Foundation, either version 3 of the
 * License, or any later version.
 * <p/>
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
 * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details. A copy of the GNU Lesser General Public License
 * is distributed along with this program and can be found at
 * <http://www.gnu.org/licenses/lgpl.html>.
 */
package ddf.catalog.impl.operations;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.tika.detect.DefaultProbDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.codice.ddf.platform.util.InputValidation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import ddf.catalog.content.data.ContentItem;
import ddf.catalog.content.data.impl.ContentItemImpl;
import ddf.catalog.data.Attribute;
import ddf.catalog.data.AttributeDescriptor;
import ddf.catalog.data.AttributeInjector;
import ddf.catalog.data.DefaultAttributeValueRegistry;
import ddf.catalog.data.Metacard;
import ddf.catalog.data.MetacardType;
import ddf.catalog.data.impl.AttributeImpl;
import ddf.catalog.impl.FrameworkProperties;
import ddf.catalog.source.IngestException;
import ddf.mime.MimeTypeResolutionException;

/**
 * Support class for working with {@code Metacard}s for the {@code CatalogFrameworkImpl}.
 * <p>
 * This class contains methods for management/manipulation for metacards for the CFI and its
 * support classes. No operations/support methods should be added to this class except in support
 * of CFI, specific to metacards.
 */
public class OperationsMetacardSupport {
    private static final Logger LOGGER = LoggerFactory.getLogger(OperationsMetacardSupport.class);

    //
    // Injected properties
    //
    private final FrameworkProperties frameworkProperties;

    private final MetacardFactory metacardFactory;

    public OperationsMetacardSupport(FrameworkProperties frameworkProperties,
            MetacardFactory metacardFactory) {
        this.frameworkProperties = frameworkProperties;
        this.metacardFactory = metacardFactory;
    }

    /**
     * Processes input metacard, injecting attributes as defined by the {@code injectors}.
     *
     * @param original  the input metacard
     * @param injectors the list of injectors to apply
     * @return the metacard, updated with any extra attributes as defined by the {@code injectors}
     */
    Metacard applyInjectors(Metacard original, List<AttributeInjector> injectors) {
        Metacard metacard = original;
        for (AttributeInjector injector : injectors) {
            metacard = injector.injectAttributes(metacard);
        }
        return metacard;
    }

    void generateMetacardAndContentItems(List<ContentItem> incomingContentItems,
            Map<String, Metacard> metacardMap, List<ContentItem> contentItems,
            Map<String, Map<String, Path>> tmpContentPaths) throws IngestException {
        for (ContentItem contentItem : incomingContentItems) {
            try {
                Path tmpPath = null;
                long size;
                try (InputStream inputStream = contentItem.getInputStream()) {
                    if (inputStream == null) {
                        throw new IngestException(
                                "Could not copy bytes of content message.  Message was NULL.");
                    }

                    String sanitizedFilename =
                            InputValidation.sanitizeFilename(contentItem.getFilename());
                    tmpPath = Files.createTempFile(FilenameUtils.getBaseName(sanitizedFilename),
                            FilenameUtils.getExtension(sanitizedFilename));
                    Files.copy(inputStream, tmpPath, StandardCopyOption.REPLACE_EXISTING);
                    size = Files.size(tmpPath);

                    final String key = contentItem.getId();
                    Map<String, Path> pathAndQualifiers = tmpContentPaths.get(key);

                    if (pathAndQualifiers == null) {
                        pathAndQualifiers = new HashMap<>();
                        pathAndQualifiers.put(contentItem.getQualifier(), tmpPath);
                        tmpContentPaths.put(key, pathAndQualifiers);
                    } else {
                        pathAndQualifiers.put(contentItem.getQualifier(), tmpPath);
                    }

                } catch (IOException e) {
                    if (tmpPath != null) {
                        FileUtils.deleteQuietly(tmpPath.toFile());
                    }
                    throw new IngestException("Could not copy bytes of content message.", e);
                }
                String mimeTypeRaw = contentItem.getMimeTypeRawData();
                mimeTypeRaw = guessMimeType(mimeTypeRaw, contentItem.getFilename(), tmpPath);

                if (!InputValidation.checkForClientSideVulnerableMimeType(mimeTypeRaw)) {
                    throw new IngestException("Unsupported mime type.");
                }

                String fileName = updateFileExtension(mimeTypeRaw, contentItem.getFilename());
                Metacard metacard = metacardFactory.generateMetacard(mimeTypeRaw,
                        contentItem.getId(),
                        fileName,
                        tmpPath);
                metacardMap.put(metacard.getId(), metacard);

                ContentItem generatedContentItem = new ContentItemImpl(metacard.getId(),
                        StringUtils.isNotEmpty(contentItem.getQualifier()) ? contentItem.getQualifier() : "",
                        com.google.common.io.Files.asByteSource(tmpPath.toFile()),
                        mimeTypeRaw,
                        fileName,
                        size,
                        metacard);
                contentItems.add(generatedContentItem);
            } catch (Exception e) {
                tmpContentPaths.values()
                        .stream()
                        .flatMap(id -> id.values()
                                .stream())
                        .forEach(path -> FileUtils.deleteQuietly(path.toFile()));
                tmpContentPaths.clear();
                throw new IngestException("Could not create metacard.", e);
            }
        }
    }

    /**
     * Updates any empty metacard attributes with those defined in the
     * {@link DefaultAttributeValueRegistry}.
     *
     * @param metacard the metacard to update with default attribute values
     */
    void setDefaultValues(Metacard metacard) {
        MetacardType metacardType = metacard.getMetacardType();
        DefaultAttributeValueRegistry registry =
                frameworkProperties.getDefaultAttributeValueRegistry();

        metacardType.getAttributeDescriptors()
                .stream()
                .map(AttributeDescriptor::getName)
                .filter(attributeName -> hasNoValue(metacard.getAttribute(attributeName)))
                .forEach(attributeName -> {
                    registry.getDefaultValue(metacardType.getName(), attributeName)
                            .ifPresent(defaultValue -> metacard.setAttribute(new AttributeImpl(
                                    attributeName,
                                    defaultValue)));
                });
    }

    private boolean hasNoValue(Attribute attribute) {
        return attribute == null || attribute.getValue() == null;
    }

    private String updateFileExtension(String mimeTypeRaw, String fileName) {
        String extension = FilenameUtils.getExtension(fileName);
        if (ContentItem.DEFAULT_FILE_NAME.equals(fileName) && !ContentItem.DEFAULT_MIME_TYPE.equals(
                mimeTypeRaw) || StringUtils.isEmpty(extension)) {
            try {
                extension = frameworkProperties.getMimeTypeMapper()
                        .getFileExtensionForMimeType(mimeTypeRaw);
                if (StringUtils.isNotEmpty(extension)) {
                    fileName = FilenameUtils.removeExtension(fileName);
                    fileName += extension;
                }
            } catch (MimeTypeResolutionException e) {
                LOGGER.debug("Unable to guess file extension for mime type.", e);
            }
        }
        return fileName;
    }

    // package-private for unit testing
    String guessMimeType(String mimeTypeRaw, String fileName, Path tmpContentPath)
            throws IOException {
        if (ContentItem.DEFAULT_MIME_TYPE.equals(mimeTypeRaw)) {
            try (InputStream inputStreamMessageCopy = com.google.common.io.Files.asByteSource(
                    tmpContentPath.toFile())
                    .openStream()) {
                String mimeTypeGuess = frameworkProperties.getMimeTypeMapper()
                        .guessMimeType(inputStreamMessageCopy,
                                FilenameUtils.getExtension(fileName));
                if (StringUtils.isNotEmpty(mimeTypeGuess)) {
                    mimeTypeRaw = mimeTypeGuess;
                }
            } catch (MimeTypeResolutionException e) {
                LOGGER.debug("Unable to guess mime type for file.", e);
            }
            if (ContentItem.DEFAULT_MIME_TYPE.equals(mimeTypeRaw)) {
                Detector detector = new DefaultProbDetector();
                try (InputStream inputStreamMessageCopy = TikaInputStream.get(tmpContentPath)) {
                    MediaType mediaType = detector.detect(inputStreamMessageCopy, new Metadata());
                    mimeTypeRaw = mediaType.toString();
                } catch (IOException e) {
                    LOGGER.debug("Unable to guess mime type for file.", e);
                }
            }
            if (mimeTypeRaw.equals("text/plain")) {
                try (InputStream inputStreamMessageCopy = com.google.common.io.Files.asByteSource(
                        tmpContentPath.toFile())
                        .openStream();
                        BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(
                                inputStreamMessageCopy,
                                Charset.forName("UTF-8")))) {
                    String line = bufferedReader.lines()
                            .map(String::trim)
                            .filter(StringUtils::isNotEmpty)
                            .findFirst()
                            .orElse("");

                    if (line.startsWith("<")) {
                        mimeTypeRaw = "text/xml";
                    } else if (line.startsWith("{") || line.startsWith("[")) {
                        mimeTypeRaw = "application/json";
                    }
                } catch (IOException e) {
                    LOGGER.debug("Unable to guess mime type for file.", e);
                }
            }
        }
        return mimeTypeRaw;
    }
}