/***********************************************************************************************************************
*
* Elasticsearch IMAP/Pop3 E-Mail Importer
* ==========================================
*
* Copyright (C) 2014 by Hendrik Saly (http://saly.de) and others.
*
* Contains (partially) copied code from Jörg Prante's Elasticsearch JDBC river (https://github.com/jprante/elasticsearch-river-jdbc)
*
***********************************************************************************************************************
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
***********************************************************************************************************************
*
* $Id:$
*
**********************************************************************************************************************/
package de.saly.elasticsearch.imap;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import javax.mail.Flags.Flag;
import javax.mail.Folder;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.Session;
import javax.mail.Store;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.codehaus.jackson.map.ObjectMapper;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.count.CountRequest;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.mapper.attachments.MapperAttachmentsPlugin;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
import org.elasticsearch.node.PluginAwareNode;
import org.elasticsearch.search.SearchHit;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.rules.TestName;
import com.google.common.collect.Lists;
import de.saly.elasticsearch.importer.imap.impl.IMAPImporter;
import de.saly.elasticsearch.importer.imap.support.IMAPUtils;
import de.saly.javamail.mock2.MockMailbox;
public abstract class AbstractIMAPRiverUnitTest {
protected static final ObjectMapper MAPPER = new ObjectMapper();
private static int SID;
protected static final String EMAIL_SUBJECT = "SID";
protected static final String EMAIL_TEXT = "This is a test e-mail.";
protected static final String EMAIL_TO = "esimaprivertest@localhost.com";
protected static final String USER_NAME = "es_imapriver_unittest";
protected static final String EMAIL_USER_ADDRESS = USER_NAME+"@localhost";
protected static final String USER_PASSWORD = USER_NAME;
protected static final String USER_NAME2 = "es_imapriver_unittest2";
protected static final String EMAIL_USER_ADDRESS2 = USER_NAME2+"@localhost";
protected static final String USER_PASSWORD2 = USER_NAME2;
protected static final String USER_NAME3 = "es_imapriver_unittest3";
protected static final String EMAIL_USER_ADDRESS3 = USER_NAME3+"@localhost";
protected static final String USER_PASSWORD3 = USER_NAME3;
@Rule
public TestName name = new TestName();
protected Node esSetup;
protected Node esSetup2;
protected Node esSetup3;
private IMAPImporter imapRiver;
protected final ESLogger logger = ESLoggerFactory.getLogger(this.getClass().getName());
protected AbstractIMAPRiverUnitTest() {
super();
System.setProperty("java.net.preferIPv4Stack","true");
}
protected Settings.Builder getSettingsBuilder(boolean dataNode, boolean masterNode) {
boolean local = true;
Settings.Builder builder = Settings
.settingsBuilder()
.put("path.home", ".")
.put("cluster.name", "imapriver_testcluster")
.put("index.store.fs.memory.enabled", "true")
.put("path.data", "target/data").put("path.work", "target/work").put("path.logs", "target/logs")
.put("path.conf", "target/config").put("plugin.types", "org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin").put("index.number_of_shards", "5")
.put("index.number_of_replicas", "0")
.put("node.data", dataNode)
.put("http.enabled", !local)
.put("node.local", local)
.put("http.cors.enabled", !local)
.put("node.master", masterNode)
.put(getProperties());
return builder;
}
@Before
public void setUp() throws Exception {
System.out.println("--------------------- SETUP " + name.getMethodName() + " -------------------------------------");
FileUtils.deleteQuietly(new File("target/data/"));
MockMailbox.resetAll();
// Instantiates a local node & client
esSetup = new PluginAwareNode(getSettingsBuilder(false, true).build(), (Collection) Lists.newArrayList(MapperAttachmentsPlugin.class)).start();
esSetup2 = new PluginAwareNode(getSettingsBuilder(true, false).build(), (Collection) Lists.newArrayList(MapperAttachmentsPlugin.class)).start();
esSetup3 = new PluginAwareNode(getSettingsBuilder(false, true).build(), (Collection) Lists.newArrayList(MapperAttachmentsPlugin.class)).start();
waitForGreenClusterState(esSetup.client());
}
@After
public void tearDown() throws Exception {
System.out.println("--------------------- TEARDOWN " + name.getMethodName() + " -------------------------------------");
if(imapRiver != null) {
imapRiver.close();
}
if (esSetup != null) {
esSetup.close();
}
if (esSetup2 != null) {
esSetup2.close();
}
if (esSetup3 != null) {
esSetup3.close();
}
FileUtils.deleteQuietly(new File("target/data/"));
}
protected void checkStoreForTestConnection(final Store store) {
if (!store.isConnected()) {
IMAPUtils.close(store);
throw new RuntimeException("Store not connected");
}
if (!store.getURLName().getUsername().toLowerCase().startsWith("es_imapriver_unittest")) {
IMAPUtils.close(store);
throw new RuntimeException("User " + store.getURLName().getUsername() + " belongs not to a valid test mail connection");
}
}
protected void createInitialIMAPTestdata(final Properties props, final String user, final String password, final int count,
final boolean deleteall) throws MessagingException {
final Session session = Session.getInstance(props);
final Store store = session.getStore();
store.connect(user, password);
checkStoreForTestConnection(store);
final Folder root = store.getDefaultFolder();
final Folder testroot = root.getFolder("ES-IMAP-RIVER-TESTS");
final Folder testrootl2 = testroot.getFolder("Level(2!");
if (deleteall) {
deleteMailsFromUserMailbox(props, "INBOX", 0, -1, user, password);
if (testroot.exists()) {
testroot.delete(true);
}
final Folder testrootenamed = root.getFolder("renamed_from_ES-IMAP-RIVER-TESTS");
if (testrootenamed.exists()) {
testrootenamed.delete(true);
}
}
if (!testroot.exists()) {
testroot.create(Folder.HOLDS_FOLDERS & Folder.HOLDS_MESSAGES);
testroot.open(Folder.READ_WRITE);
testrootl2.create(Folder.HOLDS_FOLDERS & Folder.HOLDS_MESSAGES);
testrootl2.open(Folder.READ_WRITE);
}
final Folder inbox = root.getFolder("INBOX");
inbox.open(Folder.READ_WRITE);
final Message[] msgs = new Message[count];
for (int i = 0; i < count; i++) {
final MimeMessage message = new MimeMessage(session);
message.setFrom(new InternetAddress(EMAIL_TO));
message.addRecipient(Message.RecipientType.TO, new InternetAddress(EMAIL_USER_ADDRESS));
message.setSubject(EMAIL_SUBJECT + "::" + i);
message.setText(EMAIL_TEXT + "::" + SID++);
message.setSentDate(new Date());
msgs[i] = message;
}
inbox.appendMessages(msgs);
testroot.appendMessages(msgs);
testrootl2.appendMessages(msgs);
IMAPUtils.close(inbox);
IMAPUtils.close(testrootl2);
IMAPUtils.close(testroot);
IMAPUtils.close(store);
}
protected void deleteMailsFromUserMailbox(final Properties props, final String folderName, final int start, final int deleteCount,
final String user, final String password) throws MessagingException {
final Store store = Session.getInstance(props).getStore();
store.connect(user, password);
checkStoreForTestConnection(store);
final Folder f = store.getFolder(folderName);
f.open(Folder.READ_WRITE);
final int msgCount = f.getMessageCount();
final Message[] m = deleteCount == -1 ? f.getMessages() : f.getMessages(start, Math.min(msgCount, deleteCount + start - 1));
int d = 0;
for (final Message message : m) {
message.setFlag(Flag.DELETED, true);
logger.info("Delete msgnum: {} with sid {}", message.getMessageNumber(), message.getSubject());
d++;
}
f.close(true);
logger.info("Deleted " + d + " messages");
store.close();
}
protected long getCount(final String index, final String type) {
logger.debug("getCount()");
esSetup.client().admin().indices().refresh(new RefreshRequest()).actionGet();
final CountResponse count = esSetup.client().count(new CountRequest(index).types(type)).actionGet();
return count.getCount();
}
protected long getCount(final List<String> indices, final String type) {
logger.debug("getCount() for index {} and type", indices, type);
esSetup.client().admin().indices().refresh(new RefreshRequest()).actionGet();
long count = 0;
for (Iterator<String> iterator = indices.iterator(); iterator.hasNext();) {
String index = (String) iterator.next();
long lcount = esSetup.client().count(new CountRequest(index).types(type)).actionGet().getCount();
logger.debug("Count for index {} (type {}) is {}", index, type, lcount);
count += lcount;
}
return count;
}
protected Properties getProperties() {
return new Properties();
}
protected String loadFile(final String file) throws IOException {
final StringWriter sw = new StringWriter();
IOUtils.copy(this.getClass().getResourceAsStream("/" + file), sw);
return sw.toString();
}
protected void putMailInMailbox(final int messages) throws MessagingException {
for (int i = 0; i < messages; i++) {
final MimeMessage message = new MimeMessage((Session) null);
message.setFrom(new InternetAddress(EMAIL_TO));
message.addRecipient(Message.RecipientType.TO, new InternetAddress(EMAIL_USER_ADDRESS));
message.setSubject(EMAIL_SUBJECT + "::" + i);
message.setText(EMAIL_TEXT + "::" + SID++);
message.setSentDate(new Date());
MockMailbox.get(EMAIL_USER_ADDRESS).getInbox().add(message);
}
logger.info("Putted " + messages + " into mailbox "+EMAIL_USER_ADDRESS);
}
protected void putMailInMailbox2(final int messages) throws MessagingException {
for (int i = 0; i < messages; i++) {
final MimeMessage message = new MimeMessage((Session) null);
message.setFrom(new InternetAddress(EMAIL_TO));
message.addRecipient(Message.RecipientType.TO, new InternetAddress(EMAIL_USER_ADDRESS2));
message.setSubject(EMAIL_SUBJECT + "::" + i);
message.setText(EMAIL_TEXT + "::" + SID++);
message.setSentDate(new Date());
MockMailbox.get(EMAIL_USER_ADDRESS2).getInbox().add(message);
}
logger.info("Putted " + messages + " into mailbox "+EMAIL_USER_ADDRESS2);
}
protected void putMailInMailbox3(final int messages) throws MessagingException {
for (int i = 0; i < messages; i++) {
final MimeMessage message = new MimeMessage((Session) null);
message.setFrom(new InternetAddress(EMAIL_TO));
message.addRecipient(Message.RecipientType.TO, new InternetAddress(EMAIL_USER_ADDRESS3));
message.setSubject(EMAIL_SUBJECT + "::" + i);
message.setText(EMAIL_TEXT + "::" + SID++);
message.setSentDate(new Date());
MockMailbox.get(EMAIL_USER_ADDRESS3).getInbox().add(message);
}
logger.info("Putted " + messages + " into mailbox "+EMAIL_USER_ADDRESS3);
}
@SuppressWarnings("unchecked")
protected void registerRiver(final String typename, final String file) throws ElasticsearchException, IOException {
imapRiver = new IMAPImporter(MAPPER.readValue(loadFile(file), Map.class), esSetup.client());
imapRiver.start();
}
protected SearchHit statusRiver(final String index) throws ElasticsearchException, IOException {
SearchResponse res = esSetup.client().prepareSearch(index).setTypes("imapriverstate").execute().actionGet();
if(res.getHits() != null && res.getHits().getHits() != null && res.getHits().getHits().length > 0) {
return res.getHits().getHits()[0];
}
return null;
}
protected void renameMailbox(final Properties props, final String folderName, final String user, final String password)
throws MessagingException {
final Store store = Session.getInstance(props).getStore();
store.connect(user, password);
checkStoreForTestConnection(store);
final Folder f = store.getFolder(folderName);
f.renameTo(store.getFolder("renamed_from_" + folderName));
logger.info("Renamed " + f.getFullName());
store.close();
}
@SuppressWarnings("unchecked")
protected Map<String, Object> settings(final String resource) throws IOException {
final InputStream in = this.getClass().getResourceAsStream(resource);
return MAPPER.readValue(in, Map.class);
}
protected void waitForGreenClusterState(final Client client) throws IOException {
waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(30), client);
}
protected void waitForCluster(final ClusterHealthStatus status, final TimeValue timeout, final Client client) throws IOException {
try {
logger.debug("waiting for cluster state {}", status.name());
final ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setWaitForStatus(status)
.setTimeout(timeout).execute().actionGet();
if (healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()
+ ", cowardly refusing to continue with operations");
} else {
logger.debug("... cluster state ok");
}
} catch (final ElasticsearchTimeoutException e) {
throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
}
}
}