/*
* Copyright 2012 Nodeable Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.streamreduce.storm;
import com.mongodb.*;
import com.streamreduce.Constants;
import com.streamreduce.core.metric.SobaMetric;
import com.streamreduce.util.PropertiesOverrideLoader;
import org.apache.log4j.Logger;
import org.bson.BSONObject;
import org.bson.types.ObjectId;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* A specialized MongoDB client specifically for Nodeable's needs.
*/
public class MongoClient {
public static final String BUSINESSDB_CONFIG_ID = "business";
public static final String MESSAGEDB_CONFIG_ID = "message";
public static final Logger logger = Logger.getLogger(MongoClient.class);
private final Map<String, DB> dbMap = new HashMap<>();
private String host;
private int port;
private String username;
private String password;
private Mongo mongo;
private static Properties databaseProperties;
static {
loadProperties();
}
private static void loadProperties() {
databaseProperties = PropertiesOverrideLoader.loadProperties("database.properties");
}
/**
* Connect to the Nodeable MongoDB instance using a properties file (<b>TEST-override.properties</b>)
* that uses the following naming convention for specifying the MongoDB host, port,
* username and password:
* <p/>
* Host : {configId}.database.host
* Port : {configId}.database.port
* Username: {configId}.database.username
* Password: {configId}.database.password
* <p/>
* <b>configId</b> is a simple id to find a configuration by.
*
* @param configId the database configuration id
*/
public MongoClient(String configId) {
String usernameKey = configId + ".database.user";
String passwordKey = configId + ".database.password";
init(databaseProperties.getProperty(configId + ".database.host"),
Integer.valueOf(databaseProperties.getProperty(configId + ".database.port")),
(databaseProperties.containsKey(usernameKey) ?
databaseProperties.getProperty(usernameKey) :
null),
(databaseProperties.containsKey(passwordKey) ?
databaseProperties.getProperty(passwordKey) :
null));
}
/**
* Connect to the Nodeable MongoDB instance using the values passed in. (Note: Username and password are used
* only when connecting to a specific database so please use an admin database or a user that is available
* across all the databases you need.
*
* @param host the host MongoDB is expected to be running on
* @param port the port MongoDB is expected to be listening on
* @param username the username to authenticate as
* @param password the password to the username above
*/
public MongoClient(String host, int port, String username, String password) {
init(host, port, username, password);
}
/**
* Returns the list of events in the Nodeable datastore after the date given, of
* all events if the date is null.
*
* @param since the date to get the events after (null means get all events)
* @return the list of events or an empty list if there are none
*/
public List<BasicDBObject> getEvents(Date since) {
DB connectionsDb = getDB("nodeablemsgdb");
BasicDBObject query = new BasicDBObject();
if (since != null) {
query.put("timestamp", new BasicDBObject("$gt", since.getTime()));
}
return asList(connectionsDb.getCollection("eventStream").find(query));
}
/**
* Returns the event for a particular object and version.
*
* @param targetId the object id of the target we're intersted in
* @param version the version of the object
* @return the object representing the event or null
*/
public Map<String, Object> getEventForTargetAndVersion(String targetId, int version) {
// Quick return
if (targetId == null) {
return null;
}
DB connectionsDb = getDB("nodeablemsgdb");
BasicDBObject query = new BasicDBObject();
query.put("targetId", new ObjectId(targetId));
query.put("metadata.targetVersion", version);
DBObject result = connectionsDb.getCollection("eventStream").findOne(query);
mapMongoToPlainJavaTypes(result);
return result != null ? result.toMap() : null;
}
/**
* Returns the list of events in the Nodeable datastore between the Dates specified by the since and before
* parameters.
*
* @param since the date (exclusive) to get the events after. A null means this parameter is ignored.
* @param until the date (inclusive) to get the events before. A null means this parameter is ignored.
* @return the list of events or an empty list if there are none
*/
public List<BasicDBObject> getEvents(Date since, Date until) {
DB connectionsDb = getDB("nodeablemsgdb");
QueryBuilder queryBuilder = QueryBuilder.start();
if (since != null) {
queryBuilder.and("timestamp").greaterThan(since.getTime());
}
if (until != null) {
queryBuilder.and("timestamp").lessThanEquals(until.getTime());
}
DBObject query = queryBuilder.get();
return asList(connectionsDb.getCollection("eventStream").find(query));
}
/**
* Returns the event with the given id.
*
* @param eventId the event id
* @return the {@link BasicDBObject} representing the event or null if not found
*/
public BasicDBObject getEvent(String eventId) {
DB connectionsDb = getDB("nodeablemsgdb");
DBCollection eventCollection = connectionsDb.getCollection("eventStream");
return (BasicDBObject) eventCollection.findOne(new ObjectId(eventId));
}
/**
* Returns the list of connections in the Nodeable datastore.
*
* @return the list of connections or an empty list if there are none
*/
public List<BasicDBObject> getConnections() {
DB connectionsDb = getDB("nodeabledb");
return asList(connectionsDb.getCollection("connections").find());
}
/**
* Returns a single connection with the given id.
*
* @param connectionId the id of the connection to be returned
* @return {@link BasicDBObject}
*/
public BasicDBObject getConnection(String connectionId) {
DB connectionsDb = getDB("nodeablemsgdb");
DBCollection eventCollection = connectionsDb.getCollection("connections");
return (BasicDBObject) eventCollection.findOne(new ObjectId(connectionId));
}
/**
* Returns the inventory items for the cloud connection with the given id.
*
* @param connectionId the cloud connection id
* @return a list of {@link BasicDBObject} representing each cloud inventory item
*/
public List<BasicDBObject> getCloudInventoryItems(String connectionId) {
return getInventoryItems("cloudInventoryItems", connectionId);
}
/**
* Returns the inventory items for the project hosting connection with the given id.
*
* @param connectionId the project hosting connection id
* @return a list of {@link BasicDBObject} representing each project hosting inventory item
*/
public List<BasicDBObject> getProjectHostingInventoryItems(String connectionId) {
return getInventoryItems("projectHostingInventoryItems", connectionId);
}
/**
* Reads the last processed event date of the spout.
*
* @param spoutName name of the spout
* @return last processed event date
*/
public long readLastProcessedEventDate(String spoutName) {
DB connectionsDb = getDB("nodeablemsgdb");
DBCollection eventCollection = connectionsDb.getCollection("spoutLastProcessedDate");
BasicDBObject query = new BasicDBObject();
query.put("spoutName", spoutName);
DBObject obj = eventCollection.findOne(query);
if (obj != null) {
return (Long) obj.get("lastProcessedEventDate");
} else {
return -1;
}
}
/**
* Updates the collection that tracks the last processed event date of the spouts.
*
* @param spoutName name of the spout
* @param lastProcessedEventDate last processed event date
*/
public void updateLastProcessedEventDate(String spoutName, long lastProcessedEventDate) {
DB connectionsDb = getDB("nodeablemsgdb");
DBCollection eventCollection = connectionsDb.getCollection("spoutLastProcessedDate");
BasicDBObject query = new BasicDBObject();
query.put("spoutName", spoutName);
BasicDBObject update = new BasicDBObject();
update.put("spoutName", spoutName);
update.put("lastProcessedEventDate", lastProcessedEventDate);
eventCollection.findAndModify(query, null, null, false, update, false, true);
}
/**
* Takes the information passed in and creates a {@link BasicDBObject} and writes it to the appropriate
* account-specific inbox.
*
* @param metricAccount the metric account
* @param metricName the metric name
* @param metricType the metric type
* @param metricTimestamp the metric timestamp
* @param metricValue the metric value
* @param metricGranularity the metric granularity
* @param metricCriteria the metric criteria
* @param metricAVGY the metric average/mean
* @param metricSTDDEV the metric standard deviation
* @param metricDIFF the metric diff
* @param metricMIN the metric minimum value seen
* @param metricMAX the metric maximum value seen
* @return Map containing a single entry where the key is the collection the metric was created and the value
* being the newly created {@link BasicDBObject}
*/
public Map<String, BasicDBObject> writeMetric(String metricAccount, String metricName, String metricType,
Long metricTimestamp, Float metricValue, Long metricGranularity,
Map<String, String> metricCriteria,
Float metricAVGY, Float metricSTDDEV, Float metricDIFF,
Float metricMIN, Float metricMAX, Boolean metricIsAnomaly) {
DB metricsDB = getDB("nodeablemsgdb");
String collectionName = Constants.METRIC_COLLECTION_PREFIX + metricAccount;
DBCollection metricsCollection = metricsDB.getCollection(collectionName);
BasicDBObject metric = new BasicDBObject();
metric.put("metricName", metricName);
metric.put("metricType", metricType);
metric.put("metricTimestamp", metricTimestamp);
metric.put("metricValue", metricValue);
metric.put("metricGranularity", metricGranularity);
metric.put("metricCriteria", metricCriteria);
metric.put("metricAVGY", metricAVGY);
metric.put("metricSTDDEV", metricSTDDEV);
metric.put("metricDIFF", metricDIFF);
metric.put("metricMIN", metricMIN);
metric.put("metricMAX", metricMAX);
metric.put("metricIsAnomaly", metricIsAnomaly);
metricsCollection.insert(metric);
Map<String, BasicDBObject> result = new HashMap<>();
result.put(collectionName, metric);
return result;
}
/**
* Persists the <code>metric</code> to the account-specific collection as a BasicDBObject.
*
* @param metric persisted metric object
* @return Map containing the collection name as the key and the BasicDBObject representation of the metric as the value
*/
public Map<String, BasicDBObject> writeMetric(SobaMetric metric) {
DB metricsDB = getDB("nodeablemsgdb");
String collectionName = Constants.METRIC_COLLECTION_PREFIX + metric.getStream().getAccountId();
DBCollection metricsCollection = metricsDB.getCollection(collectionName);
BasicDBObject persistedMetric = new BasicDBObject();
persistedMetric.append("timestamp", metric.getTimestamp())
.append("stream",
new BasicDBObject()
.append("accountId", metric.getStream().getAccountId())
.append("connectionId", metric.getStream().getConnectionId())
.append("inventoryItemId", metric.getStream().getInventoryItemId()))
.append("type", metric.getType().getId().toString())
.append("anomaly", metric.isAnomaly())
.append("granularity", metric.getGranularity().name())
.append("metricValue",
new BasicDBObject()
.append("mode", metric.getValue().getMode().name())
.append("type", metric.getValue().getType().name())
.append("value", metric.getValue().getValue())
.append("stddev", metric.getValue().getStddev())
.append("mean", metric.getValue().getMean())
.append("diff", metric.getValue().getDiff())
.append("min", metric.getValue().getMin())
.append("max", metric.getValue().getMax())
);
metricsCollection.insert(persistedMetric);
Map<String, BasicDBObject> result = new HashMap<>();
result.put(collectionName, persistedMetric);
return result;
}
/**
* Returns the metrics for a given account.
*
* @param metricAccount the metric account
* @return the metrics
*/
public List<BasicDBObject> getMetrics(String metricAccount) {
DB metricsDB = getDB("nodeablemsgdb");
String collectionName = Constants.METRIC_COLLECTION_PREFIX + metricAccount;
DBCollection metricsCollection = metricsDB.getCollection(collectionName);
return asList(metricsCollection.find());
}
/**
* Returns the last two metrics for a given account, metricName and granularity.
* eg: db.Inbox_4f8c34d3cea02afbc4aa8ce8.find({"metricName":"INVENTORY_ITEM_RESOURCE_USAGE.4f8c3e50cea02afbc4aa8f49.NetworkOut.average","metricGranularity":86400000}).sort({"metricTimestamp":-1}).limit(2);
*
* @param metricAccount the metric account
* @param metricName the metric name
* @param metricGranularity the metric granularity
* @return the last 2 metrics
*/
public List<Map<String, Object>> getLastTwoTuples(String metricAccount, String metricName, long metricGranularity) {
DB metricsDB = getDB("nodeablemsgdb");
String collectionName = Constants.METRIC_COLLECTION_PREFIX + metricAccount;
DBCollection metricsCollection = metricsDB.getCollection(collectionName);
BasicDBObject query = new BasicDBObject();
query.put("metricName", metricName);
query.put("metricGranularity", metricGranularity);
List<Map<String, Object>> list = new ArrayList<>();
DBCursor cursor = metricsCollection.find(query).sort(new BasicDBObject("metricTimestamp", -1)).limit(2);
while (cursor.hasNext()) {
DBObject obj = cursor.next();
mapMongoToPlainJavaTypes(obj);
list.add(obj.toMap());
}
return list;
}
/**
* Simple helper to create a list of {@link BasicDBObject} from a {@link DBCursor}.
*
* @param cursor the cursor to create a list from
* @return the list of items or an empty list of the query returned zero results
*/
private List<BasicDBObject> asList(DBCursor cursor) {
List<BasicDBObject> list = new ArrayList<>();
while (cursor.hasNext()) {
list.add((BasicDBObject) cursor.next());
}
cursor.close();
return list;
}
/**
* Returns the list of inventory items in the datastore based on the
* connection id and collection name.
*
* @param collectionName the collection name
* @param connectionId the connection id
* @return the list of inventory items, an empty list if there are no inventory items
*/
private List<BasicDBObject> getInventoryItems(String collectionName, String connectionId) {
DB db = getDB("nodeabledb");
BasicDBObject query = new BasicDBObject();
query.put("connection.$id", new ObjectId(connectionId));
return asList(db.getCollection(collectionName).find(query));
}
/**
* Initializes the {@link Mongo} object.
*
* @param host the host MongoDB is expected to be running on
* @param port the port MongoDB is expected to be listening on
* @param username the username to authenticate as
* @param password the password to the username above
*/
private void init(String host, int port, String username, String password) {
this.host = host;
this.port = port;
this.username = username;
this.password = password;
logger.info("Mongo Client Connecting to " + host + ":" + port);
try {
MongoOptions options = new MongoOptions();
options.autoConnectRetry = true;
//options.connectionsPerHost = 50;
options.connectTimeout = 1500;
options.socketTimeout = 60000;
options.threadsAllowedToBlockForConnectionMultiplier = 1000;
//options.connectionsPerHost = 50; // * 5, so up to 250 can wait before it dies
this.mongo = new Mongo(new ServerAddress(host, port), options);
} catch (UnknownHostException e) {
MongoException me = new MongoException("Unable to connect to Mongo at " +
host + ":" + port, e);
logger.error(me.getMessage(), me);
throw me;
}
}
/**
* Returns the {@link DB} object associated with the database name and
* configuration id or null if one does not exist. Authentication is
* performed on the {@link DB} prior to returning if necessary.
*
* @param dbName the database name
* @return the database for the given name
*/
private DB getDB(String dbName) {
DB db = dbMap.get(dbName);
if (db != null) {
return db;
} else {
db = mongo.getDB(dbName);
}
db = authenticateToDbIfNecessary(dbName, db);
dbMap.put(dbName, db);
return db;
}
private DB authenticateToDbIfNecessary(String dbName, DB db) {
// Authenticate if necessary
if (username != null && password != null) {
if (db.authenticate(username, password.toCharArray())) {
logger.debug("Successfully authenticated to MongoDB database (" + dbName +
") as " + username + " on " + host + ":" + port);
} else {
db = mongo.getDB("admin");
if (db.authenticate(username, password.toCharArray())) {
logger.debug("Successfully authenticated to MongoDB database (" + dbName +
") as admin " + username + " on " + host + ":" + port);
db = mongo.getDB(dbName);
} else {
throw new MongoException("Unable to authenticate to MongoDB using " +
username + "@" + dbName + " or " +
username + "@admin" + " on " + host +
":" + port + ".");
}
}
}
return db;
}
public static void mapMongoToPlainJavaTypes(BSONObject obj) {
if (obj == null) {
return;
}
for (String key : obj.keySet()) {
Object val = obj.get(key);
if (val instanceof ObjectId) {
obj.put(key, ((ObjectId) val).toString());
} else if (val instanceof BasicDBObject) {
mapMongoToPlainJavaTypes((BasicDBObject) val);
obj.put(key, ((BasicDBObject) val).toMap());
} else if (val instanceof BasicDBList) {
mapMongoToPlainJavaTypes((BasicDBList) val);
obj.put(key, new HashSet<Object>(((BasicDBList) val).toMap().values()));
}
}
}
}