ConsumeEWS.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.nifi.processors.email;

import microsoft.exchange.webservices.data.autodiscover.IAutodiscoverRedirectionUrl;
import microsoft.exchange.webservices.data.core.ExchangeService;
import microsoft.exchange.webservices.data.core.PropertySet;
import microsoft.exchange.webservices.data.core.enumeration.misc.ExchangeVersion;
import microsoft.exchange.webservices.data.core.enumeration.property.BodyType;
import microsoft.exchange.webservices.data.core.enumeration.property.WellKnownFolderName;
import microsoft.exchange.webservices.data.core.enumeration.search.FolderTraversal;
import microsoft.exchange.webservices.data.core.enumeration.search.LogicalOperator;
import microsoft.exchange.webservices.data.core.enumeration.search.SortDirection;
import microsoft.exchange.webservices.data.core.enumeration.service.ConflictResolutionMode;
import microsoft.exchange.webservices.data.core.enumeration.service.DeleteMode;
import microsoft.exchange.webservices.data.core.service.folder.Folder;
import microsoft.exchange.webservices.data.core.service.item.EmailMessage;
import microsoft.exchange.webservices.data.core.service.item.Item;
import microsoft.exchange.webservices.data.core.service.schema.EmailMessageSchema;
import microsoft.exchange.webservices.data.core.service.schema.FolderSchema;
import microsoft.exchange.webservices.data.core.service.schema.ItemSchema;
import microsoft.exchange.webservices.data.credential.ExchangeCredentials;
import microsoft.exchange.webservices.data.credential.WebCredentials;
import microsoft.exchange.webservices.data.property.complex.FileAttachment;
import microsoft.exchange.webservices.data.search.FindFoldersResults;
import microsoft.exchange.webservices.data.search.FindItemsResults;
import microsoft.exchange.webservices.data.search.FolderView;
import microsoft.exchange.webservices.data.search.ItemView;
import microsoft.exchange.webservices.data.search.filter.SearchFilter;
import org.apache.commons.mail.EmailAttachment;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.HtmlEmail;
import org.apache.commons.mail.MultiPartEmail;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.mail.Address;
import javax.mail.Flags;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.internet.MimeMessage;
import javax.mail.util.ByteArrayDataSource;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;

@InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN)
@CapabilityDescription("Consumes messages from Microsoft Exchange using Exchange Web Services. "
        + "The raw-bytes of each received email message are written as contents of the FlowFile")
@Tags({ "Email", "EWS", "Exchange", "Get", "Ingest", "Ingress", "Message", "Consume" })
public class ConsumeEWS extends AbstractProcessor {
    public static final PropertyDescriptor USER = new PropertyDescriptor.Builder()
            .name("user")
            .displayName("User Name")
            .description("User Name used for authentication and authorization with Email server.")
            .required(true)
            .expressionLanguageSupported(true)
            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
            .build();
    public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder()
            .name("password")
            .displayName("Password")
            .description("Password used for authentication and authorization with Email server.")
            .required(true)
            .expressionLanguageSupported(true)
            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
            .sensitive(true)
            .build();
    public static final PropertyDescriptor FOLDER = new PropertyDescriptor.Builder()
            .name("folder")
            .displayName("Folder")
            .description("Email folder to retrieve messages from (e.g., INBOX)")
            .required(true)
            .expressionLanguageSupported(true)
            .defaultValue("INBOX")
            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
            .build();
    public static final PropertyDescriptor FETCH_SIZE = new PropertyDescriptor.Builder()
            .name("fetch.size")
            .displayName("Fetch Size")
            .description("Specify the maximum number of Messages to fetch per call to Email Server.")
            .required(true)
            .expressionLanguageSupported(true)
            .defaultValue("10")
            .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
            .build();
    public static final PropertyDescriptor SHOULD_DELETE_MESSAGES = new PropertyDescriptor.Builder()
            .name("delete.messages")
            .displayName("Delete Messages")
            .description("Specify whether mail messages should be deleted after retrieval.")
            .required(true)
            .allowableValues("true", "false")
            .defaultValue("false")
            .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
            .build();
    static final PropertyDescriptor CONNECTION_TIMEOUT = new PropertyDescriptor.Builder()
            .name("connection.timeout")
            .displayName("Connection timeout")
            .description("The amount of time to wait to connect to Email server")
            .required(true)
            .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
            .expressionLanguageSupported(true)
            .defaultValue("30 sec")
            .build();
    public static final PropertyDescriptor EXCHANGE_VERSION = new PropertyDescriptor.Builder()
            .name("mail-ews-version")
            .displayName("Exchange Version")
            .description("What version of Exchange Server the server is running.")
            .required(true)
            .allowableValues(ExchangeVersion.values())
            .defaultValue(ExchangeVersion.Exchange2010_SP2.name())
            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
            .build();

    public static final PropertyDescriptor EWS_URL = new PropertyDescriptor.Builder()
            .name("ews-url")
            .displayName("EWS URL")
            .description("URL of the EWS Endpoint. Required if Autodiscover is false.")
            .required(false)
            .addValidator(StandardValidators.URL_VALIDATOR)
            .build();

    public static final PropertyDescriptor USE_AUTODISCOVER = new PropertyDescriptor.Builder()
            .name("ews-autodiscover")
            .displayName("Auto Discover URL")
            .description("Whether or not to use the Exchange email address to Autodiscover the EWS endpoint URL.")
            .required(true)
            .allowableValues("true","false")
            .defaultValue("true")
            .build();

    public static final PropertyDescriptor SHOULD_MARK_READ = new PropertyDescriptor.Builder()
            .name("ews-mark-as-read")
            .displayName("Mark Messages as Read")
            .description("Specify if messages should be marked as read after retrieval.")
            .required(true)
            .allowableValues("true", "false")
            .defaultValue("true")
            .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
            .build();

    static final Relationship REL_SUCCESS = new Relationship.Builder()
            .name("success")
            .description("All messages that are the are successfully received from Email server and converted to FlowFiles are routed to this relationship")
            .build();

    final protected List<PropertyDescriptor> DESCRIPTORS;

    final protected Set<Relationship> RELATIONSHIPS;

    protected final Logger logger = LoggerFactory.getLogger(this.getClass());

    protected volatile BlockingQueue<Message> messageQueue;

    protected volatile String displayUrl;

    protected volatile ProcessSession processSession;

    protected volatile boolean shouldSetDeleteFlag;

    protected volatile String folderName;
    protected volatile int fetchSize;

    public ConsumeEWS(){
        final Set<Relationship> relationshipSet = new HashSet<>();
        relationshipSet.add(REL_SUCCESS);

        RELATIONSHIPS = relationshipSet;

        final List<PropertyDescriptor> descriptors = new ArrayList<>();

        descriptors.add(USER);
        descriptors.add(PASSWORD);
        descriptors.add(FOLDER);
        descriptors.add(FETCH_SIZE);
        descriptors.add(SHOULD_DELETE_MESSAGES);
        descriptors.add(CONNECTION_TIMEOUT);
        descriptors.add(EXCHANGE_VERSION);
        descriptors.add(EWS_URL);
        descriptors.add(USE_AUTODISCOVER);
        descriptors.add(SHOULD_MARK_READ);

        DESCRIPTORS = descriptors;
    }

    @Override
    public Set<Relationship> getRelationships() {
        return RELATIONSHIPS;
    }

    @Override
    protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
        return DESCRIPTORS;
    }

    @Override
    public void onTrigger(ProcessContext context, ProcessSession processSession) throws ProcessException {
        if(this.messageQueue == null){
            int fetchSize = context.getProperty(FETCH_SIZE).evaluateAttributeExpressions().asInteger();
            this.messageQueue = new ArrayBlockingQueue<>(fetchSize);
        }

        this.folderName = context.getProperty(FOLDER).getValue();

        Message emailMessage = this.receiveMessage(context);
        if (emailMessage != null) {
            this.transfer(emailMessage, context, processSession);
        } else {
            //No new messages found, yield the processor
            context.yield();
        }
    }

    protected ExchangeService initializeIfNecessary(ProcessContext context) throws ProcessException {
        ExchangeVersion ver = ExchangeVersion.valueOf(context.getProperty(EXCHANGE_VERSION).getValue());
        ExchangeService service = new ExchangeService(ver);

        final String timeoutInMillis = String.valueOf(context.getProperty(CONNECTION_TIMEOUT).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS));
        service.setTimeout(Integer.parseInt(timeoutInMillis));

        String userEmail = context.getProperty(USER).getValue();
        String password = context.getProperty(PASSWORD).getValue();

        ExchangeCredentials credentials = new WebCredentials(userEmail, password);
        service.setCredentials(credentials);

        Boolean useAutodiscover = context.getProperty(USE_AUTODISCOVER).asBoolean();
        if(useAutodiscover){
            try {
                service.autodiscoverUrl(userEmail, new RedirectionUrlCallback());
            } catch (Exception e) {
                throw new ProcessException("Failure setting Autodiscover URL from email address.", e);
            }
        } else {
            String ewsURL = context.getProperty(EWS_URL).getValue();
            try {
                service.setUrl(new URI(ewsURL));
            } catch (URISyntaxException e) {
                throw new ProcessException("Failure setting EWS URL.", e);
            }
        }

        return service;
    }

    @Override
    protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
        return new PropertyDescriptor.Builder()
                .description("Specifies the value for '" + propertyDescriptorName + "' Java Mail property.")
                .name(propertyDescriptorName).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).dynamic(true)
                .build();
    }

    /**
     * Return the target receivere's mail protocol (e.g., imap, pop etc.)
     */
    protected String getProtocol(ProcessContext processContext) {
        return "ews";
    }

    /**
     * Fills the internal message queue if such queue is empty. This is due to
     * the fact that per single session there may be multiple messages retrieved
     * from the email server (see FETCH_SIZE).
     */
    protected void fillMessageQueueIfNecessary(ProcessContext context) throws ProcessException {
        if (this.messageQueue.isEmpty()) {
            ExchangeService service = this.initializeIfNecessary(context);
            boolean deleteOnRead = context.getProperty(SHOULD_DELETE_MESSAGES).getValue().equals("true");
            boolean markAsRead = context.getProperty(SHOULD_MARK_READ).getValue().equals("true");

            try {
                //Get Folder
                Folder folder = getFolder(service);

                ItemView view = new ItemView(messageQueue.remainingCapacity());
                view.getOrderBy().add(ItemSchema.DateTimeReceived, SortDirection.Ascending);

                SearchFilter sf = new SearchFilter.SearchFilterCollection(LogicalOperator.And, new SearchFilter.IsEqualTo(EmailMessageSchema.IsRead, false));
                FindItemsResults<Item> findResults = service.findItems(folder.getId(), sf, view);

                if(findResults == null || findResults.getItems().size()== 0){
                    return;
                }

                service.loadPropertiesForItems(findResults, PropertySet.FirstClassProperties);

                for (Item item : findResults) {
                    EmailMessage ewsMessage = (EmailMessage) item;
                    messageQueue.add(parseMessage(ewsMessage));

                    if(deleteOnRead){
                        ewsMessage.delete(DeleteMode.HardDelete);
                    } else if(markAsRead){
                        ewsMessage.setIsRead(true);
                        ewsMessage.update(ConflictResolutionMode.AlwaysOverwrite);
                    }
                }

                service.close();
            } catch (Exception e) {
                throw new ProcessException("Failed retrieving new messages from EWS.", e);
            }
        }
    }

    protected Folder getFolder(ExchangeService service) {
        Folder folder;
        if(folderName.equals("INBOX")){
            try {
                folder = Folder.bind(service, WellKnownFolderName.Inbox);
            } catch (Exception e) {
                throw new ProcessException("Failed to bind Inbox Folder on EWS Server", e);
            }
        } else {
            FolderView view = new FolderView(10);
            view.setTraversal(FolderTraversal.Deep);
            SearchFilter searchFilter = new SearchFilter.IsEqualTo(FolderSchema.DisplayName, folderName);
            try {
                FindFoldersResults foldersResults = service.findFolders(WellKnownFolderName.Root,searchFilter, view);
                ArrayList<Folder> folderIds = foldersResults.getFolders();
                if(folderIds.size() > 1){
                    throw new ProcessException("More than 1 folder found with the name " + folderName);
                }

                folder = Folder.bind(service, folderIds.get(0).getId());
            } catch (Exception e) {
                throw new ProcessException("Search for Inbox Subfolder failed.", e);
            }
        }
        return folder;
    }

    public MimeMessage parseMessage(EmailMessage item) throws Exception {
        EmailMessage ewsMessage = item;
        final String bodyText = ewsMessage.getBody().toString();

        MultiPartEmail mm;

        if(ewsMessage.getBody().getBodyType() == BodyType.HTML){
            mm = new HtmlEmail().setHtmlMsg(bodyText);
        } else {
            mm = new MultiPartEmail();
            mm.setMsg(bodyText);
        }
        mm.setHostName("NiFi-EWS");
        //from
        mm.setFrom(ewsMessage.getFrom().getAddress());
        //to recipients
        ewsMessage.getToRecipients().forEach(x->{
            try {
                mm.addTo(x.getAddress());
            } catch (EmailException e) {
                throw new ProcessException("Failed to add TO recipient.", e);
            }
        });
        //cc recipients
        ewsMessage.getCcRecipients().forEach(x->{
            try {
                mm.addCc(x.getAddress());
            } catch (EmailException e) {
                throw new ProcessException("Failed to add CC recipient.", e);
            }
        });
        //subject
        mm.setSubject(ewsMessage.getSubject());
        //sent date
        mm.setSentDate(ewsMessage.getDateTimeSent());
        //add message headers
        ewsMessage.getInternetMessageHeaders().forEach(x-> mm.addHeader(x.getName(), x.getValue()));

        //Any attachments
        if(ewsMessage.getHasAttachments()){
            ewsMessage.getAttachments().forEach(x->{
                try {
                    FileAttachment file = (FileAttachment)x;
                    file.load();

                    ByteArrayDataSource bds = new ByteArrayDataSource(file.getContent(), file.getContentType());

                    mm.attach(bds,file.getName(), "", EmailAttachment.ATTACHMENT);
                } catch (MessagingException e) {
                    e.printStackTrace();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            });
        }

        mm.buildMimeMessage();
        return mm.getMimeMessage();
    }

    /**
     * Disposes the message by converting it to a {@link FlowFile} transferring
     * it to the REL_SUCCESS relationship.
     */
    private void transfer(Message emailMessage, ProcessContext context, ProcessSession processSession) {
        long start = System.nanoTime();
        FlowFile flowFile = processSession.create();

        flowFile = processSession.append(flowFile, new OutputStreamCallback() {
            @Override
            public void process(final OutputStream out) throws IOException {
                try {
                    emailMessage.writeTo(out);
                } catch (MessagingException e) {
                    throw new IOException(e);
                }
            }
        });

        long executionDuration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);

        String fromAddressesString = "";
        try {
            Address[] fromAddresses = emailMessage.getFrom();
            if (fromAddresses != null) {
                fromAddressesString = Arrays.asList(fromAddresses).toString();
            }
        } catch (MessagingException e) {
            this.logger.warn("Faild to retrieve 'From' attribute from Message.");
        }

        processSession.getProvenanceReporter().receive(flowFile, this.displayUrl, "Received message from " + fromAddressesString, executionDuration);
        this.getLogger().info("Successfully received {} from {} in {} millis", new Object[]{flowFile, fromAddressesString, executionDuration});
        processSession.transfer(flowFile, REL_SUCCESS);

        try {
            emailMessage.setFlag(Flags.Flag.DELETED, this.shouldSetDeleteFlag);
        } catch (MessagingException e) {
            this.logger.warn("Failed to set DELETE Flag on the message, data duplication may occur.");
        }
    }

    /**
     * Receives message from the internal queue filling up the queue if
     * necessary.
     */
    protected Message receiveMessage(ProcessContext context) {
        Message emailMessage = null;
        try {
            this.fillMessageQueueIfNecessary(context);
            emailMessage = this.messageQueue.poll(1, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            context.yield();
            this.logger.error("Failed retrieving messages from EWS.", e);
            Thread.currentThread().interrupt();
            this.logger.debug("Current thread is interrupted");
        }
        return emailMessage;
    }



    @OnStopped
    public void stop(ProcessContext processContext) {
        this.flushRemainingMessages(processContext);
    }

    /**
     * Will flush the remaining messages when this processor is stopped.
     */
    protected void flushRemainingMessages(ProcessContext processContext) {
        Message emailMessage;
        try {
            while ((emailMessage = this.messageQueue.poll(1, TimeUnit.MILLISECONDS)) != null) {
                this.transfer(emailMessage, processContext, this.processSession);
                this.processSession.commit();
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            this.logger.debug("Current thread is interrupted");
        }
    }

    static class RedirectionUrlCallback implements IAutodiscoverRedirectionUrl {
        public boolean autodiscoverRedirectionUrlValidationCallback(
                String redirectionUrl) {
            return redirectionUrl.toLowerCase().startsWith("https://");
        }
    }
}