/**
* Yobi, Project Hosting SW
*
* Copyright 2014 NAVER Corp.
* http://yobi.io
*
* @Author Yi EungJun
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package mailbox;
import com.sun.mail.imap.IMAPFolder;
import com.sun.mail.imap.IMAPMessage;
import info.schleichardt.play2.mailplugin.Mailer;
import mailbox.exceptions.IllegalDetailException;
import mailbox.exceptions.MailHandlerException;
import models.OriginalEmail;
import models.Project;
import models.Property;
import models.User;
import models.enumeration.Operation;
import models.resource.Resource;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.mail.HtmlEmail;
import org.joda.time.DateTime;
import play.Logger;
import play.api.i18n.Lang;
import play.i18n.Messages;
import utils.AccessControl;
import utils.Config;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.mail.Address;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.internet.InternetAddress;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* A set of methods to process incoming emails.
*
* See {@link MailboxService} for more detailed rules to process the emails.
*/
class EmailHandler {
/**
* Fetches new emails from the given IMAP folder and process them.
*
* @param folder
* @throws MessagingException
*/
static void handleNewMessages(IMAPFolder folder) throws MessagingException {
Long lastUIDValidity = Property.getLong(Property.Name.MAILBOX_LAST_UID_VALIDITY);
Long lastSeenUID = Property.getLong(Property.Name.MAILBOX_LAST_SEEN_UID);
long uidValidity = folder.getUIDValidity();
// Get new messages and handle them
if (lastUIDValidity != null
&& lastUIDValidity.equals(uidValidity)
&& lastSeenUID != null) {
// Use the next uid instead of folder.LASTUID which possibly be
// smaller than lastSeenUID + 1.
handleMessages(folder, folder.getMessagesByUID(lastSeenUID + 1,
folder.getUIDNext()));
}
Property.set(Property.Name.MAILBOX_LAST_UID_VALIDITY, uidValidity);
}
/**
* Processes the given emails.
*
* @param folder
* @param messages
*/
static void handleMessages(IMAPFolder folder, Message[] messages) {
handleMessages(folder, Arrays.asList(messages));
}
private EmailHandler() {
// You don't need to instantiate this class because this class is just
// a set of static methods.
}
private static List<String> parseMessageIds(String headerValue) {
// in-reply-to = "In-Reply-To:" 1*msg-id CRLF
// references = "References:" 1*msg-id CRLF
// msg-id = [CFWS] "<" id-left "@" id-right ">" [CFWS]
// CFWS = (1*([FWS] comment) [FWS]) / FWS
// comment = "(" *([FWS] ccontent) [FWS] ")"
// ccontent = ctext / quoted-pair / comment
// FWS = ([*WSP CRLF] 1*WSP) / obs-FWS
// obs-FWS = 1*WSP *(CRLF 1*WSP)
// WSP = SP / HTAB
// ctext = %d33-39 / ; Printable US-ASCII
// %d42-91 / ; characters not including
// %d93-126 / ; "(", ")", or "\"
// obs-ctext
List<String> result = new ArrayList<>();
String cfws = "[^<]*(?:\\([^\\(]*\\))?[^<]*";
Pattern pattern = Pattern.compile(cfws + "(<[^>]*>)" + cfws);
Matcher matcher = pattern.matcher(headerValue);
while(matcher.find()) {
result.add(matcher.group(1));
}
return result;
}
private static void handleMessages(final IMAPFolder folder, List<Message> messages) {
// Sort messages by uid; If they are not sorted, it is possible to miss
// a email as a followed example:
//
// 1. Yobi fetches two messages with uid of 1, 3 and 2.
// 2. Yobi handles a message with uid of 1 and update lastseenuid to 1.
// 3. Yobi handles a message with uid of 3 and update lastseenuid to 3.
// 4. **Yobi Shutdown Abnormally**
// 5. The system administrator restarts Yobi.
// 6. Yobi fetches messages with uid larger than 3, the value of the
// lastseenuid; It means that **the message with uid of 2 will be
// never handled!**
Collections.sort(messages, new Comparator<Message>() {
@Override
public int compare(Message m1, Message m2) {
try {
return Long.compare(folder.getUID(m1), folder.getUID(m2));
} catch (MessagingException e) {
play.Logger.warn(
"Failed to compare uids of " + m1 + " and " + m2 +
" while sorting messages by the uid; " +
"There is some remote chance of loss of " +
"mail requests.");
return 0;
}
}
});
for (Message msg : messages) {
handleMessage((IMAPMessage) msg);
}
}
private static void handleMessage(@Nonnull IMAPMessage msg) {
Exception exception = null;
long startTime = System.currentTimeMillis();
User author;
try {
// Ignore auto-replied emails to avoid suffering from tons of
// vacation messages. For more details about auto-replied emails,
// see https://tools.ietf.org/html/rfc3834
if (isAutoReplied(msg)) {
return;
}
} catch (MessagingException e) {
play.Logger.warn(
"Failed to determine whether the email is auto-replied or not: " + msg, e);
}
try {
// Ignore the email if there is an email with the same id. It occurs
// quite frequently because mail servers send an email twice if the
// email has two addresses differ from each other: e.g.
// yobi+my/proj@mail.com and yobi+your/proj@mail.com.
OriginalEmail sameMessage =
OriginalEmail.finder.where().eq("messageId", msg.getMessageID()).findUnique();
if (sameMessage != null) {
// Warn if the older email was handled one hour or more ago. Because it is
// quite long time so that possibly the ignored email is actually
// new one which should be handled.
if (sameMessage.getHandledDate().before(new DateTime().minusHours(1).toDate())) {
String warn = String.format("This email '%s' is ignored because an email with" +
" the same id '%s' was already handled at '%s'",
msg, sameMessage.messageId, sameMessage.getHandledDate());
play.Logger.warn(warn);
}
return;
}
} catch (MessagingException e) {
play.Logger.warn(
"Failed to determine whether the email is duplicated or not: " + msg, e);
}
InternetAddress[] senderAddresses;
try {
senderAddresses = (InternetAddress[]) msg.getFrom();
} catch (Exception e) {
play.Logger.error("Failed to get senders from an email", e);
return;
}
if (senderAddresses == null || senderAddresses.length == 0) {
play.Logger.warn("This email has no sender: " + msg);
return;
}
for (InternetAddress senderAddress : senderAddresses) {
List<String> errors = new ArrayList<>();
author = User.findByEmail(senderAddress.getAddress());
if (author.isAnonymous()) {
continue;
}
try {
createResources(msg, author, errors);
} catch (MailHandlerException e) {
exception = e;
errors.add(e.getMessage());
} catch (Exception e) {
exception = e;
String shortDescription;
try {
shortDescription = IMAPMessageUtil.asString(msg);
} catch (MessagingException e1) {
shortDescription = msg.toString();
}
play.Logger.warn("Failed to process an email: " + shortDescription, e);
errors.add("Unexpected error occurs");
}
if (errors.size() > 0) {
String username = senderAddress.getPersonal();
String emailAddress = senderAddress.getAddress();
String helpMessage = getHelpMessage(
Lang.apply(author.getPreferredLanguage()), username, errors);
reply(msg, username, emailAddress, helpMessage);
}
try {
log(msg, startTime, exception);
} catch (MessagingException e) {
play.Logger.warn("Failed to log mail request", e);
}
try {
MailboxService.updateLastSeenUID(msg);
} catch (MessagingException e) {
play.Logger.warn("Failed to update the lastSeenUID", e);
}
}
}
private static class MailHeader {
private final IMAPMessage message;
private final String name;
public MailHeader(@Nonnull IMAPMessage message, @Nonnull String name) {
this.message = message;
this.name = name;
}
public boolean containsIgnoreCase(@Nonnull String expectedValue) throws MessagingException {
String[] values = message.getHeader(name);
if (values == null) {
return false;
}
for (String value : values) {
int semicolon = value.indexOf(';');
if (semicolon >= 0) {
value = value.substring(0, semicolon);
}
if (value.trim().equalsIgnoreCase(expectedValue)) {
return true;
}
}
return false;
}
}
/**
* @param message
* @return true if the given message looks like auto-replied.
* @throws MessagingException
*/
private static boolean isAutoReplied(IMAPMessage message) throws MessagingException {
return new MailHeader(message, "Auto-Submitted").containsIgnoreCase("auto-replied")
|| new MailHeader(message, "X-Naver-Absent").containsIgnoreCase("yes");
}
private static void createResources(IMAPMessage msg, User sender, List<String> errors)
throws MessagingException, IOException, MailHandlerException, NoSuchAlgorithmException {
// Find all threads by message-ids in In-Reply-To and References headers in the email.
Set<Resource> threads = getThreads(msg);
// Note: It is possible that an email creates many resources.
for (Project project : getProjects(msg, sender, errors)) {
boolean hasCommented = false;
// If there is a related thread, we assume the author wants to comment the thread.
for (Resource thread : threads) {
if (thread.getProject().id.equals(project.id)) {
switch(thread.getType()) {
case COMMENT_THREAD:
CreationViaEmail.saveReviewComment(msg, thread);
break;
case ISSUE_POST:
case BOARD_POST:
CreationViaEmail.saveComment(msg, thread);
break;
}
hasCommented = true;
}
}
// If there is no related thread, we assume the author wants to create new issue.
if (!hasCommented) {
CreationViaEmail.saveIssue(msg, project);
}
}
}
private static Set<Project> getProjects(IMAPMessage msg, User sender, List<String> errors) throws MessagingException {
Set<Project> projects = new HashSet<>();
for (EmailAddressWithDetail address : getMailAddressesToYobi(msg.getAllRecipients())) {
Project project;
String detail = address.getDetail();
if (StringUtils.isEmpty(detail)) {
// TODO: Do we need send help message?
continue;
}
Lang lang = Lang.apply(sender.getPreferredLanguage());
if (StringUtils.equalsIgnoreCase(detail, "help")) {
reply(msg, sender, getHelpMessage(lang, sender));
continue;
}
try {
project = getProjectFromDetail(detail);
} catch (IllegalDetailException e) {
errors.add(Messages.get(lang, "viaEmail.error.email", address.toString()));
continue;
}
if (project == null ||
!AccessControl.isAllowed(sender, project.asResource(), Operation.READ)) {
errors.add(Messages.get(lang, "viaEmail.error.forbidden.or.notfound",
address.toString()));
continue;
}
projects.add(project);
}
return projects;
}
/**
* Returns the threads to which the given message is sent.
*
* The threads are determined by message-ids in In-Reply-To and References
* headers and the detail part of the recipients' email addresses.
*
* @param msg
* @return
* @throws MessagingException
*/
private static Set<Resource> getThreads(IMAPMessage msg) throws MessagingException {
// Get message-ids from In-Reply-To and References headers.
Set<String> messageIds = new HashSet<>();
String inReplyTo = msg.getInReplyTo();
if (inReplyTo != null) {
messageIds.addAll(parseMessageIds(inReplyTo));
}
for (String references : ArrayUtils.nullToEmpty(msg.getHeader("References"))) {
if (references != null) {
messageIds.addAll(parseMessageIds(references));
}
}
// Find threads by the message-id.
Set<Resource> threads = new HashSet<>();
for (String messageId : messageIds) {
for (Resource resource : findResourcesByMessageId(messageId)) {
switch (resource.getType()) {
case COMMENT_THREAD:
case ISSUE_POST:
case BOARD_POST:
threads.add(resource);
break;
case REVIEW_COMMENT:
threads.add(resource.getContainer());
break;
default:
Logger.info(
"Cannot comment a resource of unknown type: " + resource);
break;
}
}
}
for (EmailAddressWithDetail address : getMailAddressesToYobi(msg.getAllRecipients())) {
Resource thread = getResourceFromDetail(address.getDetail());
if (thread != null) {
threads.add(thread);
}
}
return threads;
}
private static @Nullable Resource getResourceFromDetail(@Nullable String detail) {
if (detail == null) {
return null;
}
// detail = <owner>/<project>/<path>
// path = <resource-type>/<resource-id>
String[] segments = detail.split("/", 3);
if (segments.length < 3) {
return null;
}
return Resource.findByPath(segments[2]);
}
private static Project getProjectFromDetail(String detail) throws IllegalDetailException {
String[] segments = detail.split("/");
if (segments.length < 2) {
throw new IllegalDetailException();
}
return Project.findByOwnerAndProjectName(segments[0], segments[1]);
}
private static String getHelpMessage(Lang lang, String username, List<String> errors) {
String help = "";
String paragraphSeparator = "\n\n";
String sampleProject = "dlab/hive";
EmailAddressWithDetail address = new EmailAddressWithDetail(Config.getEmailFromImap());
address.setDetail("dlab/hive/issue");
help += Messages.get(lang, "viaEmail.help.hello", username);
if (errors != null && errors.size() > 0) {
help += paragraphSeparator;
String error;
String messageKey;
if (errors.size() > 1) {
error = "\n* " + StringUtils.join(errors, "\n* ");
messageKey = "viaEmail.help.errorMultiLine";
} else {
error = errors.get(0);
messageKey = "viaEmail.help.errorSingleLine";
}
help += Messages.get(lang, messageKey, Config.getSiteName(), error);
}
help += paragraphSeparator;
help += Messages.get(lang, "viaEmail.help.intro", Config.getSiteName());
help += paragraphSeparator;
help += Messages.get(lang, "viaEmail.help.description", sampleProject, address);
help += paragraphSeparator;
help += Messages.get(lang, "viaEmail.help.bye", Config.getSiteName());
return help;
}
private static String getHelpMessage(Lang lang, User to) {
return getHelpMessage(lang, to, null);
}
private static String getHelpMessage(Lang lang, User to, List<String> errors) {
return getHelpMessage(lang, to.name, errors);
}
private static void reply(IMAPMessage origin, String username, String emailAddress,
String msg) {
final HtmlEmail email = new HtmlEmail();
try {
email.setFrom(Config.getEmailFromSmtp(), Config.getSiteName());
email.addTo(emailAddress, username);
String subject;
if (!origin.getSubject().toLowerCase().startsWith("re:")) {
subject = "Re: " + origin.getSubject();
} else {
subject = origin.getSubject();
}
email.setSubject(subject);
email.setTextMsg(msg);
email.setCharset("utf-8");
email.setSentDate(new Date());
email.addHeader("In-Reply-To", origin.getMessageID());
email.addHeader("References", origin.getMessageID());
Mailer.send(email);
String escapedTitle = email.getSubject().replace("\"", "\\\"");
String logEntry = String.format("\"%s\" %s", escapedTitle, email.getToAddresses());
play.Logger.of("mail").info(logEntry);
} catch (Exception e) {
Logger.warn("Failed to send an email: "
+ email + "\n" + ExceptionUtils.getStackTrace(e));
}
}
private static void reply(IMAPMessage origin, User to, String msg) {
reply(origin, to.name, to.email, msg);
}
private static Set<EmailAddressWithDetail> getMailAddressesToYobi(Address[] addresses) {
Set<EmailAddressWithDetail> addressesToYobi = new HashSet<>();
if (addresses != null) {
for (Address recipient : addresses) {
EmailAddressWithDetail address = new EmailAddressWithDetail(((InternetAddress) recipient).getAddress());
if (address.isToYobi()) {
addressesToYobi.add(address);
}
}
}
return addressesToYobi;
}
/**
* Log a message for an E-mail request.
*
* @param message An email
* @param startTime the time in milliseconds when the request is received
*/
private static void log(@Nonnull IMAPMessage message, long startTime,
Exception exception) throws MessagingException {
String time = ((Long) startTime != null) ?
((System.currentTimeMillis() - startTime) + "ms") : "-";
String entry = String.format("%s %s %s",
IMAPMessageUtil.asString(message),
exception == null ? "SUCCESS" : "FAILED",
time);
if (exception != null &&
!(exception instanceof MailHandlerException)) {
Logger.of("mail.in").error(entry, exception);
} else {
Logger.of("mail.in").info(entry);
}
}
/**
* Finds resources by the given message id.
*
* If there are resources created via an email which matches the message
* id, returns the resources, else finds and returns resources which matches
* the resource path taken from the id-left part of the message id.
*
* The format of message-id is defined by RFC 5322 as follows:
*
* msg-id = [CFWS] "<" id-left "@" id-right ">" [CFWS]
*
* @param messageId
* @return the set of resources
*/
@Nonnull
public static Set<Resource> findResourcesByMessageId(String messageId) {
Set<Resource> resources = new HashSet<>();
Set<OriginalEmail> originalEmails = OriginalEmail.finder.where().eq
("messageId", messageId).findSet();
if (originalEmails.size() > 0) {
for (OriginalEmail originalEmail : originalEmails) {
resources.add(Resource.get(originalEmail.resourceType, originalEmail.resourceId));
}
return resources;
}
try {
String resourcePath = IMAPMessageUtil.getIdLeftFromMessageId(messageId);
Resource resource = Resource.findByPath(resourcePath);
if (resource != null) {
resources.add(resource);
}
} catch (Exception e) {
Logger.info(
"Error while finding a resource by message-id '" + messageId + "'", e);
}
return resources;
}
}