/*
* See the NOTICE file distributed with this work for additional
* information regarding copyright ownership.
*
* This is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of
* the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this software; if not, write to the Free
* Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
* 02110-1301 USA, or see the FSF site: http://www.fsf.org.
*/
package org.xwiki.contrib.mailarchive.internal;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.Part;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.xwiki.bridge.DocumentAccessBridge;
import org.xwiki.component.annotation.Component;
import org.xwiki.component.manager.ComponentLookupException;
import org.xwiki.component.manager.ComponentManager;
import org.xwiki.component.phase.Initializable;
import org.xwiki.component.phase.InitializationException;
import org.xwiki.context.Execution;
import org.xwiki.context.ExecutionContext;
import org.xwiki.contrib.mail.IMailComponent;
import org.xwiki.contrib.mail.IMailReader;
import org.xwiki.contrib.mail.IStoreManager;
import org.xwiki.contrib.mail.MailItem;
import org.xwiki.contrib.mail.SourceConnectionErrors;
import org.xwiki.contrib.mail.internal.FolderItem;
import org.xwiki.contrib.mail.internal.JavamailMessageParser;
import org.xwiki.contrib.mail.source.IMailSource;
import org.xwiki.contrib.mail.source.SourceType;
import org.xwiki.contrib.mailarchive.IMASource;
import org.xwiki.contrib.mailarchive.IMAUser;
import org.xwiki.contrib.mailarchive.IMailArchive;
import org.xwiki.contrib.mailarchive.IMailArchiveConfiguration;
import org.xwiki.contrib.mailarchive.IMailingList;
import org.xwiki.contrib.mailarchive.IType;
import org.xwiki.contrib.mailarchive.LoadingSession;
import org.xwiki.contrib.mailarchive.exceptions.MailArchiveException;
import org.xwiki.contrib.mailarchive.internal.data.IFactory;
import org.xwiki.contrib.mailarchive.internal.data.MailDescriptor;
import org.xwiki.contrib.mailarchive.internal.data.MailStore;
import org.xwiki.contrib.mailarchive.internal.data.Server;
import org.xwiki.contrib.mailarchive.internal.data.TopicDescriptor;
import org.xwiki.contrib.mailarchive.internal.threads.IMessagesThreader;
import org.xwiki.contrib.mailarchive.internal.threads.ThreadableMessage;
import org.xwiki.contrib.mailarchive.timeline.ITimeLineGenerator;
import org.xwiki.contrib.mailarchive.utils.DecodedMailContent;
import org.xwiki.contrib.mailarchive.utils.IMailUtils;
import org.xwiki.contrib.mailarchive.utils.ITextUtils;
import org.xwiki.contrib.mailarchive.utils.internal.TextUtils;
import org.xwiki.contrib.mailarchive.xwiki.IPersistence;
import org.xwiki.contrib.mailarchive.xwiki.internal.XWikiPersistence;
import org.xwiki.environment.Environment;
import org.xwiki.query.Query;
import org.xwiki.query.QueryException;
import org.xwiki.query.QueryManager;
import com.xpn.xwiki.XWiki;
import com.xpn.xwiki.XWikiContext;
import com.xpn.xwiki.XWikiException;
import com.xpn.xwiki.doc.XWikiDocument;
import com.xpn.xwiki.objects.BaseObject;
/**
* Implementation of a <tt>IMailArchive</tt> component.
*/
@Component
@Singleton
public class DefaultMailArchive implements IMailArchive, Initializable
{
public static final String MAIL_TYPE = "mail";
/**
* XWiki profile name of a non-existing user.
*/
public static final String UNKNOWN_USER = "XWiki.UserDoesNotExist";
public boolean isConfigured = false;
/** Is the component initialized ? */
private boolean isInitialized = false;
private Lock lock = new ReentrantLock();
private boolean locked = false;
// Components injected by the Component Manager
/** Provides access to documents. */
@Inject
private DocumentAccessBridge dab;
/** Provides access to the request context. */
@Inject
public Execution execution;
/** Provides access to execution environment and from it to context and old core */
@Inject
private Environment environment;
/**
* Secure query manager that performs checks on rights depending on the query being executed.
*/
@Inject
private QueryManager queryManager;
/** Provides access to log facility */
@Inject
Logger logger;
/**
* The component manager. We need it because we have to access some components dynamically based on the input
* syntax.
*/
@Inject
private ComponentManager componentManager;
/** Provides access to low-level mail api component */
@Inject
private IMailComponent mailManager;
// Other global objects
/** The XWiki context */
private XWikiContext context;
// TODO remove dependency to old core
/** The XWiki old core */
private XWiki xwiki;
/** Provides access to Mail archive configuration items */
@Inject
private IItemsManager store;
@Inject
@Named("mbox")
private IStoreManager builtinStore;
/** Factory to convert raw conf to POJO */
@Inject
private IFactory factory;
/** Provides access to the Mail archive configuration */
@Inject
private IMailArchiveConfiguration config;
@Inject
private IMessagesThreader threads;
@Inject
private ITimeLineGenerator timelineGenerator;
/** Used to persist pages for mails and topics */
@Inject
private IPersistence persistence;
/** Some utilities */
@Inject
public IMailUtils mailutils;
@Inject
public ITextUtils textUtils;
/** Already archived topics, loaded from database */
private HashMap<String, TopicDescriptor> existingTopics;
/** Already archived messages, loaded from database */
private HashMap<String, MailDescriptor> existingMessages;
/**
* {@inheritDoc}
*
* @see org.xwiki.component.phase.Initializable#initialize()
*/
@Override
public void initialize() throws InitializationException
{
try {
logger.debug("initialize updated()");
ExecutionContext context = execution.getContext();
this.context = (XWikiContext) context.getProperty("xwikicontext");
this.xwiki = this.context.getWiki();
// Initialize switchable logging for main components useful for the mail archive
logger.info("Mail archive initialized !");
logger.debug("PERMANENT DATA DIR : " + this.environment.getPermanentDirectory());
// Create dump folder
new File(this.environment.getPermanentDirectory(), "mailarchive/dump").mkdirs();
// Register custom job
// this.componentManager.registerComponent(this.componentManager.getComponentDescriptor(Job.class,
// "mailarchivejob"));
} catch (Throwable e) {
logger.error("Could not initiliaze mailarchive ", e);
e.printStackTrace();
}
this.isInitialized = true;
}
/**
* {@inheritDoc}
*
* @throws MailArchiveException
* @throws InitializationException
* @see org.xwiki.contrib.mailarchive.IMailArchive#getConfiguration()
*/
@Override
public IMailArchiveConfiguration getConfiguration() throws InitializationException, MailArchiveException
{
configure();
return this.config;
}
/**
* {@inheritDoc}
*
* @see org.xwiki.contrib.mailarchive.IMailArchive#checkSource(String)
*/
@Override
public int checkSource(final String sourcePrefsDoc)
{
XWikiDocument serverDoc = null;
try {
serverDoc = xwiki.getDocument(sourcePrefsDoc, context);
} catch (XWikiException e) {
serverDoc = null;
}
if (serverDoc == null || !dab.exists(sourcePrefsDoc)) {
logger.error("Page " + sourcePrefsDoc + " does not exist");
return SourceConnectionErrors.INVALID_PREFERENCES.getCode();
}
if (serverDoc.getObject(XWikiPersistence.CLASS_MAIL_SERVERS) != null) {
// Retrieve connection properties from prefs
Server server = factory.createMailServer(sourcePrefsDoc);
if (server == null) {
logger.warn("Could not retrieve server information from wiki page " + sourcePrefsDoc);
return SourceConnectionErrors.INVALID_PREFERENCES.getCode();
}
return checkSource(server);
} else if (serverDoc.getObject(XWikiPersistence.CLASS_MAIL_STORES) != null) {
// Retrieve connection properties from prefs
MailStore store = factory.createMailStore(sourcePrefsDoc);
if (store == null) {
logger.warn("Could not retrieve store information from wiki page " + sourcePrefsDoc);
return SourceConnectionErrors.INVALID_PREFERENCES.getCode();
}
return checkSource(store);
} else {
logger.error("Could not retrieve valid configuration object from page");
return SourceConnectionErrors.INVALID_PREFERENCES.getCode();
}
}
/**
* {@inheritDoc}
*
* @see org.xwiki.contrib.mailarchive.IMailArchive#checkSource(org.xwiki.contrib.mailarchive.LoadingSession)
*/
@Override
public Map<String, Integer> checkSource(final LoadingSession session)
{
Map<String, Integer> results = new HashMap<String, Integer>();
List<IMASource> sources = getSourcesList(session);
if (sources != null && !sources.isEmpty()) {
for (IMASource source : sources) {
if ("SERVER".equals(source.getType())) {
results.put(source.getWikiDoc(), checkSource((Server) source));
} else if ("STORE".equals(source.getType())) {
results.put(source.getWikiDoc(), checkSource((MailStore) source));
} else {
logger.error("Unknown type of source " + source.getType() + " for " + source.getId());
results.put(source.getWikiDoc(), SourceConnectionErrors.UNKNOWN_SOURCE_TYPE.getCode());
}
}
} else {
logger.warn("No Server nor Store found to check, nothing to do");
}
return results;
}
/**
* @param server
* @return
*/
public int checkSource(final Server server)
{
logger.info("Checking server " + server);
IMailReader mailReader = null;
try {
mailReader =
mailManager.getMailReader(server.getHostname(), server.getPort(), server.getProtocol(),
server.getUsername(), server.getPassword(), server.getAdditionalProperties(), server.isAutoTrustSSLCertificates());
} catch (ComponentLookupException e) {
logger.error("Could not find appropriate mail reader for server " + server.getId(), e);
return -1;
}
int nbMessages = mailReader.check(server.getFolder(), true);
logger.debug("check of server " + server.getId() + " returned " + nbMessages);
// Persist connection state
try {
persistence.updateMailServerState(server.getWikiDoc(), nbMessages);
} catch (Exception e) {
logger.info("Failed to persist server connection state", e);
}
server.setState(nbMessages);
return nbMessages;
}
/**
* @param store
* @return
*/
public int checkSource(final MailStore store)
{
logger.info("Checking store " + store);
IMailReader mailReader = null;
try {
mailReader = mailManager.getStoreManager(store.getFormat(), store.getLocation());
} catch (ComponentLookupException e) {
logger.error("Could not find appropriate mail reader for store " + store.getId(), e);
return SourceConnectionErrors.INVALID_PREFERENCES.getCode();
}
int nbMessages = mailReader.check(store.getFolder(), true);
logger.debug("check of server " + store.getId() + " returned " + nbMessages);
// Persist connection state
try {
persistence.updateMailStoreState(store.getWikiDoc(), nbMessages);
} catch (Exception e) {
logger.info("Failed to persist server connection state", e);
}
store.setState(nbMessages);
return nbMessages;
}
@Override
public ArrayList<FolderItem> getFolderTree(final String sourcePrefsDoc)
{
ArrayList<FolderItem> folderTree = null;
XWikiDocument serverDoc = null;
try {
serverDoc = xwiki.getDocument(sourcePrefsDoc, context);
} catch (XWikiException e) {
serverDoc = null;
}
if (serverDoc == null || !dab.exists(sourcePrefsDoc)) {
logger.error("Page " + sourcePrefsDoc + " does not exist");
return folderTree;
}
if (serverDoc.getObject(XWikiPersistence.CLASS_MAIL_SERVERS) != null) {
// Retrieve connection properties from prefs
Server server = factory.createMailServer(sourcePrefsDoc);
if (server == null) {
logger.warn("Could not retrieve server information from wiki page " + sourcePrefsDoc);
return folderTree;
}
return getFolderTree(server);
} else if (serverDoc.getObject(XWikiPersistence.CLASS_MAIL_STORES) != null) {
// Retrieve connection properties from prefs
MailStore store = factory.createMailStore(sourcePrefsDoc);
if (store == null) {
logger.warn("Could not retrieve store information from wiki page " + sourcePrefsDoc);
return folderTree;
}
return getFolderTree(store);
} else {
logger.error("Could not retrieve valid configuration object from page");
return folderTree;
}
}
public ArrayList<FolderItem> getFolderTree(final MailStore store)
{
logger.info("Checking store " + store);
IMailReader mailReader = null;
try {
mailReader = mailManager.getStoreManager(store.getFormat(), store.getLocation());
} catch (ComponentLookupException e) {
logger.warn("Could not find appropriate mail reader for store " + store.getId(), e);
}
ArrayList<FolderItem> folderTree = new ArrayList<FolderItem>();
try {
folderTree = mailReader.getFolderTree();
} catch (MessagingException e) {
logger.warn("Failed to retrieve folders from store " + store.getId(), e);
}
logger.debug("folder tree of store " + store.getId() + " returned " + folderTree);
return folderTree;
}
public ArrayList<FolderItem> getFolderTree(final Server server)
{
logger.info("Checking server " + server);
IMailReader mailReader = null;
try {
mailReader = mailManager.getMailReader(server.getHostname(), server.getPort(), server.getProtocol(), server.getUsername(), server.getPassword(), server.getAdditionalProperties(), server.isAutoTrustSSLCertificates());
} catch (ComponentLookupException e) {
logger.warn("Could not find appropriate mail reader for server " + server.getId(), e);
}
ArrayList<FolderItem> folderTree = new ArrayList<FolderItem>();
try {
folderTree = mailReader.getFolderTree();
} catch (MessagingException e) {
logger.warn("Failed to retrieve folders from server " + server.getId(), e);
}
logger.debug("folder tree of server " + server.getId() + " returned " + folderTree);
return folderTree;
}
/**
* {@inheritDoc}
*
* @see org.xwiki.contrib.mailarchive.IMailArchive#computeThreads(java.lang.String)
*/
public ThreadableMessage computeThreads(final String topicId)
{
logger.debug("computeThreads(topicId={})", topicId);
ThreadableMessage threads = null;
try {
if (topicId == null) {
threads = this.threads.thread();
} else {
threads = this.threads.thread(topicId);
}
} catch (Exception e) {
logger.error("Could not compute threads", e);
}
logger.debug("computeThreads return {}", threads);
return threads;
}
@Override
public List<IMASource> getSourcesList(final LoadingSession session)
{
logger.debug("Getting sources for session {}", session);
final List<IMASource> servers = new ArrayList<IMASource>();
final Map<SourceType, String> sources = session.getSources();
boolean hasServers = false;
if (sources != null) {
for (Entry<SourceType, String> source : sources.entrySet()) {
if (SourceType.SERVER.equals(source.getKey())) {
final String prefsDoc = XWikiPersistence.SPACE_PREFS + ".Server_" + source.getValue();
Server server = factory.createMailServer(prefsDoc);
if (server != null) {
hasServers = true;
servers.add(server);
}
} else if (SourceType.STORE.equals(source.getKey())) {
final String prefsDoc = XWikiPersistence.SPACE_PREFS + ".Store_" + source.getValue();
MailStore store = factory.createMailStore(prefsDoc);
if (store != null) {
servers.add(store);
}
} else {
// This should never occur
logger.warn("Unknown type of source connection: " + source.getKey());
}
}
}
if (!hasServers && servers.isEmpty() && config.getServers() != null) {
// Empty server config means all servers
// Empty store config means no store
servers.addAll(config.getServers());
}
logger.debug("Found sources for session {} : {}", session.getId(), servers);
return servers;
}
public String computeTimeline() throws XWikiException, InitializationException, MailArchiveException, IOException
{
logger.debug("computeTimeline");
if (!this.isConfigured) {
configure();
}
String timelineFeed = timelineGenerator.compute();
if (!StringUtils.isBlank(timelineFeed)) {
File timelineFeedPath = new File(environment.getPermanentDirectory(), "mailarchive/timeline");
if (!timelineFeedPath.exists() || !timelineFeedPath.isDirectory()) {
timelineFeedPath.mkdirs();
}
FileWriter fw = new FileWriter(new File(timelineFeedPath, "TimeLineFeed.xml"), false);
fw.write(timelineFeed);
fw.close();
}
logger.debug("computeTimeline return {}" , timelineFeed);
return timelineFeed;
}
/**
* @throws InitializationException
* @throws MailArchiveException
*/
protected void configure(boolean loadTopicsAndMails) throws InitializationException, MailArchiveException
{
// Init
if (!this.isInitialized) {
initialize();
}
config.reloadConfiguration();
if (config.getItemsSpaceName() != null && !"".equals(config.getItemsSpaceName())) {
XWikiPersistence.SPACE_ITEMS = config.getItemsSpaceName();
}
if (config.isUseStore()) {
File maStoreLocation = new File(environment.getPermanentDirectory(), "mailarchive/storage");
logger.info("Local Store Location: " + maStoreLocation.getAbsolutePath());
logger.info("Local Store Provider: mstor");
try {
this.builtinStore = mailManager.getStoreManager("mbox", maStoreLocation.getAbsolutePath());
} catch (ComponentLookupException e) {
logger.error("Could not create or connect built-in store", e);
throw new InitializationException("Could not create or connect to built-in store", e);
}
}
TextUtils.setLogger(this.logger);
if (loadTopicsAndMails) {
loadTopicsAndMails();
}
this.isConfigured = true;
}
protected void configure() throws InitializationException, MailArchiveException
{
configure(true);
}
protected void loadTopicsAndMails() throws MailArchiveException
{
existingTopics = store.loadStoredTopics();
existingMessages = store.loadStoredMessages();
}
@Override
public Map<String, TopicDescriptor> getTopics() throws MailArchiveException
{
return store.loadStoredTopics();
}
@Override
public Map<String, MailDescriptor> getMails() throws MailArchiveException
{
return store.loadStoredMessages();
}
public IType getType(String name)
{
return config.getMailTypes().get(name);
}
/**
* @param m
*/
public void setMailSpecificParts(final MailItem m)
{
logger.debug("Extracting types");
try {
// Built-in types
// TODO: manage calendar built-in type, for now default is mail for all emails
m.setBuiltinType(MAIL_TYPE);
// Types
List<IType> foundTypes = mailutils.extractTypes(config.getMailTypes().values(), m);
logger.debug("Extracted types " + foundTypes);
// foundTypes.remove(getType(IType.BUILTIN_TYPE_MAIL));
if (foundTypes.size() > 0) {
for (IType foundType : foundTypes) {
logger.debug("Adding extracted type " + foundType);
m.addType(foundType.getId());
}
} /*
* else { logger.debug("No specific type found for this mail");
* m.addType(getType(IType.BUILTIN_TYPE_MAIL).getId()); }
*/
// Mailing-lists
m.setMailingLists(extractMailingListsTags(m));
// User
logger.debug("Extracting user information");
String userwiki = null;
if (config.isMatchProfiles()) {
IMAUser maUser = mailutils.parseUser(m.getFrom(), config.isMatchLdap());
userwiki = maUser.getWikiProfile();
}
if (StringUtils.isBlank(userwiki)) {
if (config.isMatchProfiles()) {
userwiki = UNKNOWN_USER;
} else {
userwiki = config.getLoadingUser();
}
}
m.setWikiuser(userwiki);
// If no topic id is provided by message, we default to message id
if (StringUtils.isBlank(m.getTopicId())) {
m.setTopicId(m.getMessageId());
}
// Compatibility: crop ids
if (config.isCropTopicIds() && m.getTopicId().length() >= 30) {
m.setTopicId(m.getTopicId().substring(0, 29));
}
} catch (Throwable t) {
logger.warn("Exception ", t);
t.printStackTrace();
}
}
@Override
public IMAUser parseUser(final String internetAddress)
{
logger.debug("parseUser {}", internetAddress);
try {
configure(false);
} catch (Exception e) {
logger.warn("parseUser: failed to configure the Mail Archive", e);
return null;
}
IMAUser user = mailutils.parseUser(internetAddress, config.isMatchLdap());
logger.debug("parseUser return {}", user);
return user;
}
/**
* @param mail
* @param confirm
* @param isAttachedMail
* @param parentMail
* @return
* @throws XWikiException
* @throws ParseException
* @throws IOException
* @throws MessagingException
*/
@Override
public MailLoadingResult loadMail(final Part mail, final boolean confirm, final boolean isAttachedMail,
final String parentMail) throws XWikiException, ParseException, MessagingException, IOException
{
MailItem m = null;
logger.debug("Parsing headers");
m = mailManager.parseHeaders(mail);
if (StringUtils.isBlank(m.getFrom())) {
logger.warn("Invalid email : missing 'from' header, skipping it");
return new MailLoadingResult(MailLoadingResult.STATUS.FAILED, null, null);
}
logger.debug("Parsing specific parts");
setMailSpecificParts(m);
// Compatibility option with old version of the mail archive
if (config.isCropTopicIds() && m.getTopicId().length() > 30) {
m.setTopicId(m.getTopicId().substring(0, 29));
}
logger.info("Parsed email " + m);
return loadMail(m, confirm, isAttachedMail, parentMail);
}
/**
* @param m
* @param confirm
* @param isAttachedMail
* @param parentMail
* @throws XWikiException
* @throws ParseException
*/
public MailLoadingResult loadMail(final MailItem m, final boolean confirm, final boolean isAttachedMail,
final String parentMail) throws XWikiException, ParseException
{
String topicDocName = null;
String messageDocName = null;
logger.debug("Loading mail content into wiki objects");
// set loading user for rights - loading user must have edit rights on IMailArchive and MailArchiveCode spaces
context.setUser(config.getLoadingUser());
logger.debug("Loading user " + config.getLoadingUser() + " set in context");
SimpleDateFormat dateFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZZZ", m.getLocale());
// Do not archive email if nodlmatch option is set, and no mailing-list is matched
logger.debug("*** isAttachedMail {}", isAttachedMail);
logger.debug("*** isNoLdMatch {}", config.isNoLdMatch());
logger.debug("*** Extracted mailing-lists {}", extractMailingListsTags(m));
if (!isAttachedMail && !config.isNoLdMatch() && CollectionUtils.isEmpty(extractMailingListsTags(m))) {
logger.info("No mailing-list matched, skipping email creation");
return new MailLoadingResult(MailLoadingResult.STATUS.NOT_MATCHING_MAILING_LISTS, null, null);
} else {
// Create a new topic if needed
String existingTopicId = "";
// we don't create new topics for attached emails
if (!isAttachedMail) {
existingTopicId =
existsTopic(m.getTopicId(), m.getTopic(), m.getReplyToId(), m.getMessageId(), m.getRefs());
if (existingTopicId == null) {
logger.debug(" did not find existing topic, creating a new one");
if (existingTopics.containsKey(m.getTopicId())) {
// Topic hack ...
logger.debug(" new topic but topicId already loaded, using messageId as new topicId");
m.setTopicId(m.getMessageId());
// FIX: "cut" properly mail history when creating a new topic
m.setReplyToId("");
existingTopicId =
existsTopic(m.getTopicId(), m.getTopic(), m.getReplyToId(), m.getMessageId(), m.getRefs());
} else {
existingTopicId = m.getTopicId();
}
logger.debug(" creating new topic");
topicDocName = createTopicPage(m, confirm);
logger.info("Saved new topic " + topicDocName);
} else if (textUtils.similarSubjects(m.getTopic(), existingTopics.get(existingTopicId).getSubject())) {
logger.debug(" topic already loaded " + m.getTopicId() + " : "
+ existingTopics.get(existingTopicId));
topicDocName = updateTopicPage(m, existingTopicId, dateFormatter, confirm);
logger.info("Updated topic " + topicDocName);
} else {
// We consider this was a topic hack : someone replied to an existing thread, but to start on
// another
// subject.
// In this case, we split, use messageId as a new topic Id, and set replyToId to empty string in
// order
// to treat this as a new topic to create.
// In order for this new thread to be correctly threaded, we search for existing topic with this new
// topicId,
// so now all new mails in this case will be attached to this new topic.
logger
.debug(" found existing topic but subjects are too different, using new messageid as topicid ["
+ m.getMessageId() + "]");
m.setTopicId(m.getMessageId());
m.setReplyToId("");
existingTopicId =
existsTopic(m.getTopicId(), m.getTopic(), m.getReplyToId(), m.getMessageId(), m.getRefs());
logger.debug(" creating new topic");
topicDocName = createTopicPage(m, confirm);
logger.info("Saved new topic from hijacked thread " + topicDocName);
}
} // if not attached email
// Create a new message if needed
if (!existingMessages.containsKey(m.getMessageId())) {
logger.info("creating new message " + m.getMessageId() + " ...");
/*
* Note : use already existing topic id if any, instead of the one from the message, to keep an easy to
* parse link between thread messages
*/
if ("".equals(existingTopicId)) {
existingTopicId = m.getTopicId();
}
// Note : correction bug of messages linked to same topic but with different topicIds
m.setTopicId(existingTopicId);
try {
String parent = parentMail;
if (StringUtils.isBlank(parentMail)) {
parent = existingTopics.get(m.getTopicId()).getFullName();
}
messageDocName = createMailPage(m, existingTopicId, isAttachedMail, parent, confirm);
logger.info("Saved new message " + messageDocName);
} catch (Exception e) {
logger.warn("Could not create mail page for " + m.getMessageId(), e);
return new MailLoadingResult(MailLoadingResult.STATUS.FAILED, topicDocName, null);
}
return new MailLoadingResult(MailLoadingResult.STATUS.SUCCESS, topicDocName, messageDocName);
} else {
// message already loaded
logger.info("Mail already loaded - checking for updates ...");
MailDescriptor msg = existingMessages.get(m.getMessageId());
logger.debug("TopicId of existing message " + msg.getTopicId() + " and of topic " + existingTopicId
+ " are different ?" + (!msg.getTopicId().equals(existingTopicId)));
if (!msg.getTopicId().equals(existingTopicId)) {
messageDocName = existingMessages.get(m.getMessageId()).getFullName();
XWikiDocument msgDoc = xwiki.getDocument(messageDocName, context);
BaseObject msgObj = msgDoc.getObject(XWikiPersistence.SPACE_CODE + ".MailClass");
msgObj.set("topicid", existingTopicId, context);
if (confirm) {
logger.debug("saving message " + m.getSubject());
persistence.saveAsUser(msgDoc, null, config.getLoadingUser(),
"Updated mail with existing topic id found");
}
logger.info("Updated message " + msgDoc.getFullName());
return new MailLoadingResult(MailLoadingResult.STATUS.SUCCESS, topicDocName, messageDocName);
}
return new MailLoadingResult(MailLoadingResult.STATUS.ALREADY_LOADED, topicDocName, messageDocName);
}
}
}
/**
* createTopicPage Creates a wiki page for a Topic.
*
* @throws XWikiException
*/
protected String createTopicPage(final MailItem m, final boolean create) throws XWikiException
{
String pageName = "T" + m.getTopic().replaceAll(" ", "");
// Materialize mailing-lists information and mail IType in Tags
List<String> taglist = extractTags(m);
String createdTopicName = persistence.createTopic(pageName, m, taglist, config.getLoadingUser(), create);
// add the existing topic created to the map
existingTopics.put(m.getTopicId(), new TopicDescriptor(createdTopicName, m.getTopic()));
return createdTopicName;
}
protected String updateTopicPage(final MailItem m, final String existingTopicId,
final SimpleDateFormat dateFormatter, final boolean create) throws XWikiException
{
logger.debug("updateTopicPage(" + m.toString() + ", existingTopicId=" + existingTopicId + ")");
final String existingTopicPage = existingTopics.get(existingTopicId).getFullName();
logger.debug("Topic page to update: " + existingTopicPage);
String updatedTopicName =
persistence.updateTopicPage(m, existingTopicPage, dateFormatter, config.getLoadingUser(), create);
existingTopics.put(m.getTopicId(), new TopicDescriptor(updatedTopicName, m.getTopic()));
return updatedTopicName;
}
protected String createMailPage(final MailItem m, final String existingTopicId, final boolean isAttachedMail,
final String parentMail, final boolean create) throws XWikiException, MessagingException, IOException
{
// Materialize mailing-lists information and mail IType in Tags
final String pageName = persistence.getMessageUniquePageName(m, isAttachedMail);
// Parse mail content
m.setMailContent(mailManager.parseContent(m.getOriginalMessage()));
List<String> taglist = extractTags(m);
// We load attachment emails first - so we can link them afterwards
List<String> attachedMailPages = loadAttachedMails(m.getMailContent().getAttachedMails(), pageName, create);
final String createdPageName =
persistence.createMailPage(m, pageName, existingTopicId, isAttachedMail, taglist, attachedMailPages,
parentMail, config.getLoadingUser(), create);
existingMessages.put(m.getMessageId(), new MailDescriptor(m.getSubject(), existingTopicId, createdPageName));
return createdPageName;
}
private List<String> loadAttachedMails(final List<Message> attachedMails, final String parentFullName,
final boolean create)
{
final List<String> attachedMailsPages = new ArrayList<String>();
if (attachedMails.size() > 0) {
logger.debug("Loading attached mails ...");
for (Message message : attachedMails) {
try {
MailLoadingResult result = loadMail(message, create, true, parentFullName);
if (result.isSuccess()) {
attachedMailsPages.add(result.getCreatedMailDocumentName());
} else {
logger.warn("Could not create attached mail " + message.getSubject());
}
} catch (Exception e) {
logger.warn("Could not create attached mail because of " + e.getMessage());
if (logger.isDebugEnabled()) {
logger.debug("Could not create attached mail ", e);
}
}
}
}
return attachedMailsPages;
}
protected List<String> extractTags(final MailItem m)
{
// Materialize mailing-lists information and mail IType in Tags
List<String> taglist = extractMailingListsTags(m);
for (String typeid : m.getTypes()) {
IType type = config.getMailTypes().get(typeid);
taglist.add(type.getName());
}
taglist.add(m.getBuiltinType());
return taglist;
}
/**
* @param m
* @return
*/
protected List<String> extractMailingListsTags(final MailItem m)
{
List<String> mailingLists = new ArrayList<String>();
for (IMailingList list : config.getMailingLists().values()) {
if (m.getFrom().contains(list.getDisplayName()) || m.getTo().contains(list.getPattern())
|| m.getCc().contains(list.getPattern())) {
// Add tag of this mailing-list to the list of tags
mailingLists.add(list.getTag());
}
}
return mailingLists;
}
/**
* Returns the topicId of already existing topic for this topic id or subject. If no topic with this id or subject
* is found, try to search for a message for wich msgid = replyid of new msg, then attach this new msg to the same
* topic. If there is no existing topic, returns null. Search topic with same subject only if inreplyto is not
* empty, meaning it's not supposed to be the first message of another topic.
*
* @param topicId
* @param topicSubject
* @param inreplyto
* @return
*/
public String existsTopic(final String topicId, final String topicSubject, final String inreplyto,
final String messageid, final String refs)
{
String foundTopicId = null;
String replyId = inreplyto;
String previous = "";
String previousSubject = topicSubject;
boolean quit = false;
// Search in existing messages for existing msg id = new reply id, and grab topic id
// search replies until root message
while (StringUtils.isNotBlank(replyId) && existingMessages.containsKey(replyId)
&& existingMessages.get(replyId) != null && !quit) {
XWikiDocument msgDoc = null;
try {
msgDoc = context.getWiki().getDocument(existingMessages.get(replyId).getFullName(), context);
} catch (XWikiException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if (msgDoc != null) {
BaseObject msgObj = msgDoc.getObject(XWikiPersistence.SPACE_CODE + ".MailClass");
if (msgObj != null) {
logger
.debug("existsTopic : message " + replyId + " is a reply to " + existingMessages.get(replyId));
if (textUtils.similarSubjects(previousSubject, msgObj.getStringValue("topicsubject"))) {
previous = replyId;
replyId = msgObj.getStringValue("inreplyto");
previousSubject = msgObj.getStringValue("topicSubject");
} else {
logger.debug("existsTopic : existing message subject is too different, exiting loop");
quit = true;
}
} else {
replyId = null;
}
} else {
replyId = null;
}
}
if (replyId != inreplyto && replyId != null) {
logger
.debug("existsTopic : found existing message that current message is a reply to, to attach to same topic id");
foundTopicId = existingMessages.get(previous).getTopicId();
logger.debug("existsTopic : Found topic id " + foundTopicId);
}
// Search in existing topics with id
if (foundTopicId == null) {
if (!StringUtils.isBlank(topicId) && existingTopics.containsKey(topicId)) {
logger.debug("existsTopic : found topic id in loaded topics");
if (textUtils.similarSubjects(topicSubject, existingTopics.get(topicId).getSubject())) {
foundTopicId = topicId;
} else {
logger.debug("... but subjects are too different");
}
}
}
// Search with references
if (foundTopicId == null) {
String xwql =
"select distinct mail.topicid from Document doc, doc.object(" + XWikiPersistence.CLASS_MAILS
+ ") as mail where mail.references like '%" + messageid + "%'";
try {
List<String> topicIds = queryManager.createQuery(xwql, Query.XWQL).execute();
// We're not supposed to find several topics related to messages having this id in references ...
if (topicIds.size() == 1) {
foundTopicId = topicIds.get(0);
}
if (topicIds.size() > 1) {
logger.warn("We should have found only one topicId instead of this list : " + topicIds
+ ", using the first found");
}
} catch (QueryException e) {
logger.warn("Issue while searching for references", e);
}
}
// Search in existing topics with exactly same subject
if (foundTopicId == null) {
for (String currentTopicId : existingTopics.keySet()) {
TopicDescriptor currentTopic = existingTopics.get(currentTopicId);
if (currentTopic.getSubject().trim().equalsIgnoreCase(topicSubject.trim())) {
logger.debug("existsTopic : found subject in loaded topics");
if (!StringUtils.isBlank(inreplyto)) {
logger.debug("existsTopic : not first message in topic, so we assume it's linked to it");
foundTopicId = currentTopicId;
} else {
logger.debug("existsTopic : found a topic but it's first message in topic");
// Note : desperate tentative to attach this message to an existing topic
// instead of creating a new one ... Sometimes replyId and refs can be
// empty even if this is a reply to something already loaded, in this
// case we just check if topicId was already loaded once, even if not
// the same topic ...
if (existingTopics.containsKey(topicId)) {
logger
.debug("existsTopic : ... but we 'saw' this topicId before, so attach to found topicId "
+ currentTopicId + " with same subject");
foundTopicId = currentTopicId;
}
if (!StringUtils.isBlank(refs)) {
logger.debug("existsTopic : ... but references are not empty, so attach to found topicId "
+ currentTopicId + " with same subject");
foundTopicId = currentTopicId;
}
}
}
}
}
return foundTopicId;
}
/**
* {@inheritDoc}
*
* @throws IOException
* @throws XWikiException
* @throws MailArchiveException
* @throws InitializationException
* @see org.xwiki.contrib.mailarchive.IMailArchive#getDecodedMailText(java.lang.String, boolean)
*/
@SuppressWarnings("deprecation")
@Override
public DecodedMailContent getDecodedMailText(final String mailPage, final boolean cut) throws IOException,
XWikiException, InitializationException, MailArchiveException
{
if (!this.isConfigured) {
configure();
}
if (!StringUtils.isBlank(mailPage)) {
XWikiDocument htmldoc = null;
try {
htmldoc = xwiki.getDocument(mailPage, this.context);
} catch (Throwable t) {
// FIXME Ugly workaround for "org.hibernate.SessionException: Session is closed!"
try {
htmldoc = xwiki.getDocument(mailPage, this.context);
} catch (Exception e) {
htmldoc = null;
}
}
if (htmldoc != null) {
BaseObject htmlobj = htmldoc.getObject("MailArchiveCode.MailClass");
if (htmlobj != null) {
String ziphtml = htmlobj.getLargeStringValue("bodyhtml");
String body = htmlobj.getLargeStringValue("body");
return (mailutils.decodeMailContent(ziphtml, body, cut));
}
}
}
return new DecodedMailContent(false, "");
}
/**
* Try to get a lock on the archive. (non-blocking)
*
* @return true if lock could be set, or false if lock is already in use.
*/
@Override
public boolean lock()
{
this.locked = this.lock.tryLock();
return this.locked;
}
/**
* Unlocks the archive.
*/
@Override
public void unlock()
{
this.lock.unlock();
this.locked = false;
}
@Override
public boolean isLocked()
{
return this.locked;
}
@Override
public void saveToInternalStore(final String serverId, final IMailSource source, final Message message)
{
// Save to internal store only if we did not already load this mail from internal store ...
if (!StringUtils.isBlank(serverId) && !builtinStore.getMailSource().equals(source)) {
try {
// Use server id as folder to avoid colliding folders from different servers
builtinStore.write(serverId, message);
logger.info("Message written to internal store");
} catch (MessagingException e) {
logger.error("Can't copy mail to local store", e);
}
}
}
@Override
public Message getFromStore(final String serverId, final String messageId)
{
Message message = null;
try {
message = builtinStore.read(serverId, messageId);
} catch (MessagingException e) {
logger.debug("Message with id {} not found from builtin store in folder {}", messageId, serverId);
}
return message;
}
@Override
public void dumpEmail(final Message message)
{
try {
final String id =
JavamailMessageParser.extractSingleHeader(message, "Message-ID").replaceAll("[^a-zA-Z0-9-_\\.]", "_");
final File emlFile = new File(environment.getPermanentDirectory(), "mailarchive/dump/" + id + ".eml");
emlFile.createNewFile();
final FileOutputStream fo = new FileOutputStream(emlFile);
message.writeTo(fo);
fo.flush();
fo.close();
logger.debug("Message dumped into {}.eml", id);
} catch (Throwable t) {
// we catch Throwable because we don't want to cause problems in debug mode
logger.debug("Could not dump message for debug", t);
}
}
}