/*
* The MIT License
*
* Copyright 2012 Sony Mobile Communications Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package com.sonyericsson.jenkins.plugins.bfa.db;
import com.mongodb.AggregationOutput;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBObject;
import com.mongodb.DBRef;
import com.mongodb.Mongo;
import com.mongodb.MongoException;
import com.sonyericsson.jenkins.plugins.bfa.Messages;
import com.sonyericsson.jenkins.plugins.bfa.graphs.FailureCauseTimeInterval;
import com.sonyericsson.jenkins.plugins.bfa.graphs.GraphFilterBuilder;
import com.sonyericsson.jenkins.plugins.bfa.model.FailureCause;
import com.sonyericsson.jenkins.plugins.bfa.model.indication.FoundIndication;
import com.sonyericsson.jenkins.plugins.bfa.statistics.FailureCauseStatistics;
import com.sonyericsson.jenkins.plugins.bfa.statistics.Statistics;
import com.sonyericsson.jenkins.plugins.bfa.utils.BfaUtils;
import com.sonyericsson.jenkins.plugins.bfa.utils.ObjectCountPair;
import hudson.Extension;
import hudson.Util;
import hudson.model.Descriptor;
import hudson.model.Run;
import hudson.util.FormValidation;
import hudson.util.Secret;
import jenkins.model.Jenkins;
import net.vz.mongodb.jackson.DBCursor;
import net.vz.mongodb.jackson.JacksonDBCollection;
import net.vz.mongodb.jackson.WriteResult;
import org.apache.commons.collections.keyvalue.MultiKey;
import org.bson.types.ObjectId;
import org.jfree.data.time.Day;
import org.jfree.data.time.Hour;
import org.jfree.data.time.Month;
import org.jfree.data.time.TimePeriod;
import org.kohsuke.stapler.DataBoundConstructor;
import org.kohsuke.stapler.QueryParameter;
import javax.naming.AuthenticationException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SimpleTimeZone;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static java.util.Arrays.asList;
/**
* Handling of the MongoDB way of saving the knowledge base.
*
* @author Tomas Westling <tomas.westling@sonyericsson.com>
*/
public class MongoDBKnowledgeBase extends KnowledgeBase {
private static final long serialVersionUID = 4984133048405390951L;
/**The name of the cause collection in the database.*/
public static final String COLLECTION_NAME = "failureCauses";
/**The name of the statistics collection in the database.*/
public static final String STATISTICS_COLLECTION_NAME = "statistics";
private static final int MONGO_DEFAULT_PORT = 27017;
/**
* Query to single out documents that doesn't have a "removed" property
*/
static final BasicDBObject NOT_REMOVED_QUERY = new BasicDBObject("_removed", new BasicDBObject("$exists", false));
private static final Logger logger = Logger.getLogger(MongoDBKnowledgeBase.class.getName());
private transient Mongo mongo;
private transient DB db;
private transient DBCollection collection;
private transient DBCollection statisticsCollection;
private transient JacksonDBCollection<FailureCause, String> jacksonCollection;
private transient JacksonDBCollection<Statistics, String> jacksonStatisticsCollection;
private transient MongoDBKnowledgeBaseCache cache;
private String host;
private int port;
private String dbName;
private String userName;
private Secret password;
private boolean enableStatistics;
private boolean successfulLogging;
/**
* Getter for the MongoDB user name.
* @return the user name.
*/
public String getUserName() {
return userName;
}
/**
* Getter for the MongoDB password.
* @return the password.
*/
public Secret getPassword() {
return password;
}
/**
* Getter for the host value.
* @return the host string.
*/
public String getHost() {
return host;
}
/**
* Getter for the port value.
* @return the port number.
*/
public int getPort() {
return port;
}
/**
* Getter for the database name value.
* @return the database name string.
*/
public String getDbName() {
return dbName;
}
/**
* Standard constructor.
* @param host the host to connect to.
* @param port the port to connect to.
* @param dbName the database name to connect to.
* @param userName the user name for the database.
* @param password the password for the database.
* @param enableStatistics if statistics logging should be enabled or not.
* @param successfulLogging if all builds should be logged to the statistics DB
*/
@DataBoundConstructor
public MongoDBKnowledgeBase(String host, int port, String dbName, String userName, Secret password,
boolean enableStatistics, boolean successfulLogging) {
this.host = host;
this.port = port;
this.dbName = dbName;
this.userName = userName;
this.password = password;
this.enableStatistics = enableStatistics;
this.successfulLogging = successfulLogging;
}
@Override
public synchronized void start() throws UnknownHostException, AuthenticationException {
initCache();
}
@Override
public synchronized void stop() {
if (cache != null) {
cache.stop();
cache = null;
}
}
/**
* Initializes the cache if it is null.
* @throws UnknownHostException if we cannot connect to the database.
* @throws AuthenticationException if we cannot authenticate towards the database.
*/
private void initCache() throws UnknownHostException, AuthenticationException {
if (cache == null) {
cache = new MongoDBKnowledgeBaseCache(getJacksonCollection());
cache.start();
}
}
/**
* @see KnowledgeBase#getCauses()
* Can throw MongoException if unknown fields exist in the database.
* @return the full list of causes.
* @throws UnknownHostException if a connection to the host cannot be made.
* @throws AuthenticationException if we cannot authenticate towards the database.
*/
@Override
public Collection<FailureCause> getCauses() throws UnknownHostException, AuthenticationException {
initCache();
return cache.getCauses();
}
/**
* @see KnowledgeBase#getCauseNames()
* Can throw MongoException if unknown fields exist in the database.
* @return the full list of the names and ids of the causes..
* @throws UnknownHostException if a connection to the host cannot be made.
* @throws AuthenticationException if we cannot authenticate towards the database.
*/
@Override
public Collection<FailureCause> getCauseNames() throws UnknownHostException, AuthenticationException {
List<FailureCause> list = new LinkedList<FailureCause>();
DBObject keys = new BasicDBObject();
keys.put("name", 1);
DBCursor<FailureCause> dbCauses = getJacksonCollection().find(NOT_REMOVED_QUERY, keys);
while (dbCauses.hasNext()) {
list.add(dbCauses.next());
}
return list;
}
@Override
public Collection<FailureCause> getShallowCauses() throws Exception {
List<FailureCause> list = new LinkedList<FailureCause>();
DBObject keys = new BasicDBObject();
keys.put("name", 1);
keys.put("description", 1);
keys.put("categories", 1);
keys.put("comment", 1);
keys.put("modifications", 1);
keys.put("lastOccurred", 1);
BasicDBObject orderBy = new BasicDBObject("name", 1);
DBCursor<FailureCause> dbCauses = getJacksonCollection().find(NOT_REMOVED_QUERY, keys);
dbCauses = dbCauses.sort(orderBy);
while (dbCauses.hasNext()) {
list.add(dbCauses.next());
}
return list;
}
@Override
public FailureCause getCause(String id) throws UnknownHostException, AuthenticationException {
FailureCause returnCase = null;
try {
returnCase = getJacksonCollection().findOneById(id);
} catch (IllegalArgumentException e) {
logger.fine("Could not find the id, returning null for id: " + id);
return returnCase;
}
return returnCase;
}
@Override
public FailureCause addCause(FailureCause cause) throws UnknownHostException, AuthenticationException {
return addCause(cause, true);
}
@Override
public FailureCause removeCause(String id) throws Exception {
BasicDBObject idq = new BasicDBObject("_id", new ObjectId(id));
BasicDBObject removedInfo = new BasicDBObject("timestamp", new Date());
removedInfo.put("by", Jenkins.getAuthentication().getName());
BasicDBObject update = new BasicDBObject("$set", new BasicDBObject("_removed", removedInfo));
FailureCause modified = getJacksonCollection().findAndModify(idq, null, null, false, update, true, false);
initCache();
cache.updateCache();
return modified;
}
/**
* Does not update the cache, used when we know we will have a lot of save/add calls all at once,
* e.g. during a convert.
*
* @param cause the FailureCause to add.
* @param doUpdate true if a cache update should be made, false if not.
*
* @return the added FailureCause.
*
* @throws UnknownHostException If a connection to the Mongo database cannot be made.
* @throws javax.naming.AuthenticationException if we cannot authenticate towards the database.
*
* @see MongoDBKnowledgeBase#addCause(FailureCause)
*/
public FailureCause addCause(FailureCause cause, boolean doUpdate) throws UnknownHostException,
AuthenticationException {
WriteResult<FailureCause, String> result = getJacksonCollection().insert(cause);
if (doUpdate) {
initCache();
cache.updateCache();
}
return result.getSavedObject();
}
@Override
public FailureCause saveCause(FailureCause cause) throws UnknownHostException, AuthenticationException {
return saveCause(cause, true);
}
/**
* Does not update the cache, used when we know we will have a lot of save/add calls all at once,
* e.g. during a convert.
*
* @param cause the FailureCause to save.
* @param doUpdate true if a cache update should be made, false if not.
*
* @return the saved FailureCause.
*
* @throws UnknownHostException If a connection to the Mongo database cannot be made.
* @throws AuthenticationException if we cannot authenticate towards the database.
*
* @see MongoDBKnowledgeBase#saveCause(FailureCause)
*/
public FailureCause saveCause(FailureCause cause, boolean doUpdate) throws UnknownHostException,
AuthenticationException {
WriteResult<FailureCause, String> result = getJacksonCollection().save(cause);
if (doUpdate) {
initCache();
cache.updateCache();
}
return result.getSavedObject();
}
@Override
public void convertFrom(KnowledgeBase oldKnowledgeBase) throws Exception {
if (oldKnowledgeBase instanceof MongoDBKnowledgeBase) {
convertFromAbstract(oldKnowledgeBase);
convertRemoved((MongoDBKnowledgeBase)oldKnowledgeBase);
} else {
for (FailureCause cause : oldKnowledgeBase.getCauseNames()) {
try {
//try finding the id in the knowledgebase, if so, update it.
if (getCause(cause.getId()) != null) {
//doing all the additions to the database first and then fetching to the cache only once.
saveCause(cause, false);
//if not found, add a new.
} else {
cause.setId(null);
addCause(cause, false);
}
//Safety net for the case that Mongo should throw anything if the id has a really weird form.
} catch (MongoException e) {
cause.setId(null);
addCause(cause, false);
}
}
initCache();
cache.updateCache();
}
}
@Override
public List<String> getCategories() throws UnknownHostException, AuthenticationException {
initCache();
return cache.getCategories();
}
/**
* Copies all causes flagged as removed from the old database to this one.
*
* @param oldKnowledgeBase the old database.
* @throws Exception if something goes wrong.
*/
protected void convertRemoved(MongoDBKnowledgeBase oldKnowledgeBase) throws Exception {
List<DBObject> removed = oldKnowledgeBase.getRemovedCauses();
DBCollection dbCollection = getJacksonCollection().getDbCollection();
for (DBObject obj : removed) {
dbCollection.save(obj);
}
}
/**
* Gets all causes flagged as removed in a "raw" JSON format.
*
* @return the list of removed causes.
* @throws Exception if so.
*/
protected List<DBObject> getRemovedCauses() throws Exception {
BasicDBObject query = new BasicDBObject("_removed", new BasicDBObject("$exists", true));
com.mongodb.DBCursor dbCursor = getJacksonCollection().getDbCollection().find(query);
List<DBObject> removed = new LinkedList<DBObject>();
while (dbCursor.hasNext()) {
removed.add(dbCursor.next());
}
return removed;
}
@Override
public boolean equals(KnowledgeBase oldKnowledgeBase) {
if (getClass().isInstance(oldKnowledgeBase)) {
MongoDBKnowledgeBase oldMongoDBKnowledgeBase = (MongoDBKnowledgeBase)oldKnowledgeBase;
return equals(oldMongoDBKnowledgeBase.getHost(), host)
&& oldMongoDBKnowledgeBase.getPort() == port
&& equals(oldMongoDBKnowledgeBase.getDbName(), dbName)
&& equals(oldMongoDBKnowledgeBase.getUserName(), userName)
&& equals(oldMongoDBKnowledgeBase.getPassword(), password)
&& this.enableStatistics == oldMongoDBKnowledgeBase.enableStatistics
&& this.successfulLogging == oldMongoDBKnowledgeBase.successfulLogging;
} else {
return false;
}
}
@Override
public boolean equals(Object other) {
if (other instanceof KnowledgeBase) {
return this.equals((KnowledgeBase)other);
} else {
return false;
}
}
/**
* Checks if two objects equal each other, both being null counts as being equal.
* @param firstObject the firstObject.
* @param secondObject the secondObject.
* @return true if equal or both null, false otherwise.
*/
public static boolean equals(Object firstObject, Object secondObject) {
if (firstObject == null) {
if (secondObject == null) {
return true;
}
return false;
}
if (secondObject == null) {
return false;
}
return secondObject.equals(firstObject);
}
@Override
public int hashCode() {
//Making checkstyle happy.
return getClass().getName().hashCode();
}
@Override
public boolean isStatisticsEnabled() {
return enableStatistics;
}
@Override
public boolean isSuccessfulLoggingEnabled() {
return successfulLogging;
}
@Override
public void saveStatistics(Statistics stat) throws UnknownHostException, AuthenticationException {
DBObject object = new BasicDBObject();
object.put("projectName", stat.getProjectName());
object.put("buildNumber", stat.getBuildNumber());
object.put("displayName", stat.getDisplayName());
object.put("master", stat.getMaster());
object.put("slaveHostName", stat.getSlaveHostName());
object.put("startingTime", stat.getStartingTime());
object.put("duration", stat.getDuration());
object.put("timeZoneOffset", stat.getTimeZoneOffset());
object.put("triggerCauses", stat.getTriggerCauses());
DBObject cause = null;
if (stat.getUpstreamCause() != null) {
cause = new BasicDBObject();
Statistics.UpstreamCause upstreamCause = stat.getUpstreamCause();
cause.put("project", upstreamCause.getUpstreamProject());
cause.put("build", upstreamCause.getUpstreamBuild());
}
object.put("upstreamCause", cause);
object.put("result", stat.getResult());
List<FailureCauseStatistics> failureCauseStatisticsList = stat.getFailureCauseStatisticsList();
addFailureCausesToDBObject(object, failureCauseStatisticsList);
getStatisticsCollection().insert(object);
}
@Override
public List<Statistics> getStatistics(GraphFilterBuilder filter, int limit)
throws UnknownHostException, AuthenticationException {
DBObject matchFields = generateMatchFields(filter);
DBCursor<Statistics> dbCursor = getJacksonStatisticsCollection().find(matchFields);
BasicDBObject buildNumberDescending = new BasicDBObject("buildNumber", -1);
dbCursor = dbCursor.sort(buildNumberDescending);
if (limit > 0) {
dbCursor = dbCursor.limit(limit);
}
return dbCursor.toArray();
}
@Override
public long getNbrOfNullFailureCauses(GraphFilterBuilder filter) {
DBObject matchFields = generateMatchFields(filter);
matchFields.put("failureCauses", null);
try {
return getStatisticsCollection().count(matchFields);
} catch (Exception e) {
logger.fine("Unable to get number of null failure causes");
e.printStackTrace();
}
return -1;
}
@Override
public Map<TimePeriod, Double> getUnknownFailureCauseQuotaPerTime(int intervalSize, GraphFilterBuilder filter) {
Map<TimePeriod, Integer> unknownFailures = new HashMap<TimePeriod, Integer>();
Map<TimePeriod, Integer> knownFailures = new HashMap<TimePeriod, Integer>();
Set<TimePeriod> periods = new HashSet<TimePeriod>();
DBObject matchFields = generateMatchFields(filter);
DBObject match = new BasicDBObject("$match", matchFields);
// Use $project to change all null failurecauses to 'false' since
// it's not possible to group by 'null':
DBObject projectFields = new BasicDBObject();
projectFields.put("startingTime", 1);
DBObject nullToFalse = new BasicDBObject("$ifNull", asList("$failureCauses", false));
projectFields.put("failureCauses", nullToFalse);
DBObject project = new BasicDBObject("$project", projectFields);
// Group by date and false/non false failure causes:
DBObject idFields = generateTimeGrouping(intervalSize);
DBObject checkNullFailureCause = new BasicDBObject("$eq", asList("$failureCauses", false));
idFields.put("isNullFailureCause", checkNullFailureCause);
DBObject groupFields = new BasicDBObject();
groupFields.put("_id", idFields);
groupFields.put("number", new BasicDBObject("$sum", 1));
DBObject group = new BasicDBObject("$group", groupFields);
AggregationOutput output;
try {
output = getStatisticsCollection().aggregate(match, project, group);
for (DBObject result : output.results()) {
DBObject groupedAttrs = (DBObject)result.get("_id");
TimePeriod period = generateTimePeriodFromResult(result, intervalSize);
periods.add(period);
int number = (Integer)result.get("number");
boolean isNullFailureCause = (Boolean)groupedAttrs.get("isNullFailureCause");
if (isNullFailureCause) {
unknownFailures.put(period, number);
} else {
knownFailures.put(period, number);
}
}
} catch (Exception e) {
logger.fine("Unable to get unknown failure cause quota per time");
e.printStackTrace();
}
Map<TimePeriod, Double> nullFailureCauseQuotas = new HashMap<TimePeriod, Double>();
for (TimePeriod timePeriod : periods) {
int unknownFailureCount = 0;
int knownFailureCount = 0;
if (unknownFailures.containsKey(timePeriod)) {
unknownFailureCount = unknownFailures.get(timePeriod);
}
if (knownFailures.containsKey(timePeriod)) {
knownFailureCount = knownFailures.get(timePeriod);
}
double quota;
if (unknownFailureCount == 0) {
quota = 0d;
} else {
quota = ((double)unknownFailureCount) / (unknownFailureCount + knownFailureCount);
}
nullFailureCauseQuotas.put(timePeriod, quota);
}
return nullFailureCauseQuotas;
}
@Override
public List<ObjectCountPair<String>> getNbrOfFailureCausesPerId(GraphFilterBuilder filter, int maxNbr) {
List<ObjectCountPair<String>> nbrOfFailureCausesPerId = new ArrayList<ObjectCountPair<String>>();
DBObject matchFields = generateMatchFields(filter);
DBObject match = new BasicDBObject("$match", matchFields);
DBObject unwind = new BasicDBObject("$unwind", "$failureCauses");
DBObject groupFields = new BasicDBObject();
groupFields.put("_id", "$failureCauses.failureCause");
groupFields.put("number", new BasicDBObject("$sum", 1));
DBObject group = new BasicDBObject("$group", groupFields);
DBObject sort = new BasicDBObject("$sort", new BasicDBObject("number", -1));
DBObject limit = null;
if (maxNbr > 0) {
limit = new BasicDBObject("$limit", maxNbr);
}
AggregationOutput output;
try {
if (limit == null) {
output = getStatisticsCollection().aggregate(match, unwind, group, sort);
} else {
output = getStatisticsCollection().aggregate(match, unwind, group, sort, limit);
}
for (DBObject result : output.results()) {
DBRef failureCauseRef = (DBRef)result.get("_id");
if (failureCauseRef != null) {
Integer number = (Integer)result.get("number");
String id = failureCauseRef.getId().toString();
nbrOfFailureCausesPerId.add(new ObjectCountPair<String>(id, number));
}
}
} catch (Exception e) {
logger.fine("Unable to get failure causes per id");
e.printStackTrace();
}
return nbrOfFailureCausesPerId;
}
@Override
public Date getLatestFailureForCause(String id) {
DBObject causeToMatch = new BasicDBObject("$ref", "failureCauses");
causeToMatch.put("$id", new ObjectId(id));
DBObject causeList = new BasicDBObject("failureCauses.failureCause", causeToMatch);
DBObject match = new BasicDBObject("$match", causeList);
DBObject sort = new BasicDBObject("$sort", new BasicDBObject("startingTime", -1));
DBObject limit = new BasicDBObject("$limit", 1);
AggregationOutput output;
try {
output = getStatisticsCollection().aggregate(match, sort, limit);
for (DBObject result : output.results()) {
Date startingTime = (Date)result.get("startingTime");
if (startingTime != null) {
return startingTime;
}
}
} catch (Exception e) {
logger.log(Level.WARNING, "Failed getting latest failure of cause", e);
}
return null;
}
@Override
public Date getCreationDateForCause(String id) {
Date creationDate;
try {
//Get the creation date using time information in MongoDB id:
creationDate = new Date(new ObjectId(id).getTime());
} catch (IllegalArgumentException e) {
logger.log(Level.WARNING, "Could not retrieve original modification", e);
creationDate = new Date(0);
}
return creationDate;
}
@Override
public void updateLastSeen(List<String> ids, Date seen) {
List<ObjectId> objectIds = new LinkedList<ObjectId>();
for (String id : ids) {
objectIds.add(new ObjectId(id));
}
DBObject match = new BasicDBObject("_id", new BasicDBObject("$in", objectIds));
DBObject set = new BasicDBObject("$set", new BasicDBObject("lastOccurred", seen));
try {
getJacksonCollection().updateMulti(match, set);
} catch (UnknownHostException e) {
logger.log(Level.WARNING, "Failed connecting to MongoDB when updating FailureCauses' last occurrence", e);
} catch (AuthenticationException e) {
logger.log(Level.WARNING, "Failed authentication when updating FailureCauses' last occurrence", e);
}
}
/**
* Generates a DBObject used for matching data as part of a MongoDb
* aggregation query.
*
* @param filter the filter to create match fields for
* @return DBObject containing fields to match
*/
private static DBObject generateMatchFieldsBase(GraphFilterBuilder filter) {
DBObject matchFields = new BasicDBObject();
if (filter != null) {
putNonNullStringValue(matchFields, "master", filter.getMasterName());
putNonNullStringValue(matchFields, "slaveHostName", filter.getSlaveName());
putNonNullStringValue(matchFields, "projectName", filter.getProjectName());
putNonNullStringValue(matchFields, "result", filter.getResult());
putNonNullBasicDBObject(matchFields, "buildNumber", "$in", filter.getBuildNumbers());
putNonNullBasicDBObject(matchFields, "startingTime", "$gte", filter.getSince());
putNonNullBasicDBObject(matchFields, "result", "$ne", filter.getExcludeResult());
}
return matchFields;
}
/**
* Generates the standard DBObject for filtering, with the additional exclusion of successful builds.
*
* @param filter the filter to create match fields for
* @return DBObject containing fields to match
*/
private static DBObject generateMatchFields(GraphFilterBuilder filter) {
DBObject matchFields = generateMatchFieldsBase(filter);
putNonNullBasicDBObject(matchFields, "result", "$ne", "SUCCESS");
return matchFields;
}
/**
* Puts argument value to the dbObject if the value is non-null.
* @param dbObject object to put value to.
* @param key the key to map the value to.
* @param value the value to set.
*/
private static void putNonNullStringValue(DBObject dbObject, String key, String value) {
if (value != null) {
dbObject.put(key, value);
}
}
/**
* Puts argument value to the dbObject if the value is non-null.
* The value will be added with an MongoDB operator, for example "$in" or "$gte".
* @param dbObject object to put value to.
* @param key the key to map the value to.
* @param operator the MongoDB operator to add together with the value.
* @param value the value to set.
*/
private static void putNonNullBasicDBObject(DBObject dbObject, String key,
String operator, Object value) {
if (value != null) {
dbObject.put(key, new BasicDBObject(operator, value));
}
}
@Override
public List<ObjectCountPair<FailureCause>> getNbrOfFailureCauses(GraphFilterBuilder filter) {
List<ObjectCountPair<String>> nbrOfFailureCausesPerId = getNbrOfFailureCausesPerId(filter, 0);
List<ObjectCountPair<FailureCause>> nbrOfFailureCauses = new ArrayList<ObjectCountPair<FailureCause>>();
try {
for (ObjectCountPair<String> countPair : nbrOfFailureCausesPerId) {
String id = countPair.getObject();
int count = countPair.getCount();
FailureCause failureCause = getCause(id);
if (failureCause != null) {
nbrOfFailureCauses.add(new ObjectCountPair<FailureCause>(failureCause, count));
}
}
} catch (Exception e) {
logger.fine("Unable to count failure causes");
e.printStackTrace();
}
return nbrOfFailureCauses;
}
@Override
public List<ObjectCountPair<String>> getFailureCauseNames(GraphFilterBuilder filter) {
List<ObjectCountPair<String>> nbrOfFailureCauseNames = new ArrayList<ObjectCountPair<String>>();
for (ObjectCountPair<FailureCause> countPair : getNbrOfFailureCauses(filter)) {
FailureCause failureCause = countPair.getObject();
if (failureCause.getName() != null) {
nbrOfFailureCauseNames.add(new ObjectCountPair<String>(failureCause.getName(), countPair.getCount()));
}
}
return nbrOfFailureCauseNames;
}
@Override
public Map<Integer, List<FailureCause>> getFailureCausesPerBuild(GraphFilterBuilder filter) {
Map<Integer, List<FailureCause>> nbrOfFailureCausesPerBuild = new HashMap<Integer, List<FailureCause>>();
DBObject matchFields = generateMatchFields(filter);
DBObject match = new BasicDBObject("$match", matchFields);
DBObject unwind = new BasicDBObject("$unwind", "$failureCauses");
DBObject groupFields = new BasicDBObject("_id", "$buildNumber");
groupFields.put("failureCauses", new BasicDBObject("$addToSet", "$failureCauses.failureCause"));
DBObject group = new BasicDBObject("$group", groupFields);
DBObject sort = new BasicDBObject("$sort", new BasicDBObject("_id", 1));
AggregationOutput output;
try {
output = getStatisticsCollection().aggregate(match, unwind, group, sort);
for (DBObject result : output.results()) {
List<FailureCause> failureCauses = new ArrayList<FailureCause>();
Integer buildNumber = (Integer)result.get("_id");
BasicDBList failureCauseRefs = (BasicDBList)result.get("failureCauses");
for (Object o : failureCauseRefs) {
DBRef failureRef = (DBRef)o;
String id = failureRef.getId().toString();
FailureCause failureCause = getCause(id);
failureCauses.add(failureCause);
}
nbrOfFailureCausesPerBuild.put(buildNumber, failureCauses);
}
} catch (Exception e) {
logger.fine("Unable to count failure causes by build");
e.printStackTrace();
}
return nbrOfFailureCausesPerBuild;
}
/**
* Generates a {@link DBObject} used for grouping data into time intervals
* @param intervalSize the interval size, should be set to Calendar.HOUR_OF_DAY,
* Calendar.DATE or Calendar.MONTH.
* @return DBObject to be used for time grouping
*/
private DBObject generateTimeGrouping(int intervalSize) {
DBObject timeFields = new BasicDBObject();
if (intervalSize == Calendar.HOUR_OF_DAY) {
timeFields.put("hour", new BasicDBObject("$hour", "$startingTime"));
}
if (intervalSize == Calendar.HOUR_OF_DAY || intervalSize == Calendar.DATE) {
timeFields.put("dayOfMonth", new BasicDBObject("$dayOfMonth", "$startingTime"));
}
timeFields.put("month", new BasicDBObject("$month", "$startingTime"));
timeFields.put("year", new BasicDBObject("$year", "$startingTime"));
return timeFields;
}
/**
* Generates a {@link TimePeriod} based on a MongoDB grouping aggregation result.
* @param result the result to interpret
* @param intervalSize the interval size, should be set to Calendar.HOUR_OF_DAY,
* Calendar.DATE or Calendar.MONTH.
* @return TimePeriod
*/
private TimePeriod generateTimePeriodFromResult(DBObject result, int intervalSize) {
BasicDBObject groupedAttrs = (BasicDBObject)result.get("_id");
int month = groupedAttrs.getInt("month");
int year = groupedAttrs.getInt("year");
Calendar c = Calendar.getInstance();
c.set(Calendar.YEAR, year);
c.set(Calendar.MONTH, month - 1);
// MongoDB timezone is UTC:
c.setTimeZone(new SimpleTimeZone(0, "UTC"));
TimePeriod period;
if (intervalSize == Calendar.HOUR_OF_DAY) {
int dayOfMonth = groupedAttrs.getInt("dayOfMonth");
c.set(Calendar.DAY_OF_MONTH, dayOfMonth);
int hour = groupedAttrs.getInt("hour");
c.set(Calendar.HOUR_OF_DAY, hour);
period = new Hour(c.getTime());
} else if (intervalSize == Calendar.DATE) {
int dayOfMonth = groupedAttrs.getInt("dayOfMonth");
c.set(Calendar.DAY_OF_MONTH, dayOfMonth);
period = new Day(c.getTime());
} else {
period = new Month(c.getTime());
}
return period;
}
@Override
public List<FailureCauseTimeInterval> getFailureCausesPerTime(int intervalSize, GraphFilterBuilder filter,
boolean byCategories) {
List<FailureCauseTimeInterval> failureCauseIntervals = new ArrayList<FailureCauseTimeInterval>();
Map<MultiKey, FailureCauseTimeInterval> categoryTable = new HashMap<MultiKey, FailureCauseTimeInterval>();
DBObject matchFields = generateMatchFields(filter);
DBObject match = new BasicDBObject("$match", matchFields);
DBObject unwind = new BasicDBObject("$unwind", "$failureCauses");
DBObject idFields = generateTimeGrouping(intervalSize);
idFields.put("failureCause", "$failureCauses.failureCause");
DBObject groupFields = new BasicDBObject();
groupFields.put("_id", idFields);
groupFields.put("number", new BasicDBObject("$sum", 1));
DBObject group = new BasicDBObject("$group", groupFields);
AggregationOutput output;
try {
output = getStatisticsCollection().aggregate(match, unwind, group);
for (DBObject result : output.results()) {
int number = (Integer)result.get("number");
TimePeriod period = generateTimePeriodFromResult(result, intervalSize);
BasicDBObject groupedAttrs = (BasicDBObject)result.get("_id");
DBRef failureRef = (DBRef)groupedAttrs.get("failureCause");
String id = failureRef.getId().toString();
FailureCause failureCause = getCause(id);
if (byCategories) {
if (failureCause.getCategories() != null) {
for (String category : failureCause.getCategories()) {
MultiKey multiKey = new MultiKey(category, period);
FailureCauseTimeInterval interval = categoryTable.get(multiKey);
if (interval == null) {
interval = new FailureCauseTimeInterval(period, category, number);
categoryTable.put(multiKey, interval);
failureCauseIntervals.add(interval);
} else {
interval.addNumber(number);
}
}
}
} else {
FailureCauseTimeInterval timeInterval = new FailureCauseTimeInterval(period, failureCause.getName(),
failureCause.getId(), number);
failureCauseIntervals.add(timeInterval);
}
}
} catch (UnknownHostException e) {
logger.fine("Unable to get failure causes by time");
e.printStackTrace();
} catch (AuthenticationException e) {
logger.fine("Unable to get failure causes by time");
e.printStackTrace();
}
return failureCauseIntervals;
}
@Override
public List<ObjectCountPair<String>> getNbrOfFailureCategoriesPerName(GraphFilterBuilder filter, int limit) {
List<ObjectCountPair<String>> nbrOfFailureCausesPerId = getNbrOfFailureCausesPerId(filter, 0);
Map<String, Integer> nbrOfFailureCategoriesPerName = new HashMap<String, Integer>();
for (ObjectCountPair<String> countPair : nbrOfFailureCausesPerId) {
String id = countPair.getObject();
int count = countPair.getCount();
FailureCause failureCause = null;
try {
failureCause = getCause(id);
} catch (Exception e) {
logger.fine("Unable to count failure causes by name");
e.printStackTrace();
}
if (failureCause != null) {
if (failureCause.getCategories() == null) {
Integer currentNbr = nbrOfFailureCategoriesPerName.get(null);
if (currentNbr == null) {
currentNbr = 0;
}
currentNbr += count;
nbrOfFailureCategoriesPerName.put(null, currentNbr);
} else {
for (String category : failureCause.getCategories()) {
Integer currentNbr = nbrOfFailureCategoriesPerName.get(category);
if (currentNbr == null) {
currentNbr = 0;
}
currentNbr += count;
nbrOfFailureCategoriesPerName.put(category, currentNbr);
}
}
}
}
List<ObjectCountPair<String>> countList = new ArrayList<ObjectCountPair<String>>();
for (Entry<String, Integer> entry : nbrOfFailureCategoriesPerName.entrySet()) {
String name = entry.getKey();
int count = entry.getValue();
countList.add(new ObjectCountPair<String>(name, count));
}
Collections.sort(countList, ObjectCountPair.countComparator());
if (limit > 0 && countList.size() > limit) {
countList = countList.subList(0, limit);
}
return countList;
}
@Override
public void removeBuildfailurecause(Run build) throws Exception {
BasicDBObject searchObj = new BasicDBObject();
searchObj.put("projectName", build.getParent().getFullName());
searchObj.put("buildNumber", build.getNumber());
searchObj.put("master", BfaUtils.getMasterName());
com.mongodb.DBCursor dbcursor = getStatisticsCollection().find(searchObj);
if (dbcursor != null && dbcursor.size() > 0) {
while (dbcursor.hasNext()) {
getStatisticsCollection().remove(dbcursor.next());
logger.log(Level.INFO, build.getDisplayName() + " build failure cause removed");
}
} else {
logger.log(Level.INFO, build.getDisplayName() + " build failure cause "
+ "value is null or initial scanning ");
}
}
/**
* Adds the FailureCauses from the list to the DBObject.
* @param object the DBObject to add to.
* @param failureCauseStatisticsList the list of FailureCauseStatistics to add.
* @throws UnknownHostException If the mongoDB host cannot be found.
* @throws AuthenticationException if the mongoDB authentication fails.
*/
private void addFailureCausesToDBObject(DBObject object, List<FailureCauseStatistics> failureCauseStatisticsList)
throws UnknownHostException, AuthenticationException {
if (failureCauseStatisticsList != null && !failureCauseStatisticsList.isEmpty()) {
List<DBObject> failureCauseStatisticsObjects = new LinkedList<DBObject>();
for (FailureCauseStatistics failureCauseStatistics : failureCauseStatisticsList) {
DBObject failureCauseStatisticsObject = new BasicDBObject();
ObjectId id = new ObjectId(failureCauseStatistics.getId());
DBRef failureCauseRef = new DBRef(getDb(), COLLECTION_NAME, id);
failureCauseStatisticsObject.put("failureCause", failureCauseRef);
List<FoundIndication> foundIndicationList = failureCauseStatistics.getIndications();
addIndicationsToDBObject(failureCauseStatisticsObject, foundIndicationList);
failureCauseStatisticsObjects.add(failureCauseStatisticsObject);
}
object.put("failureCauses", failureCauseStatisticsObjects);
}
}
/**
* Adds the indications from the list to the DBObject.
* @param object the DBObject to add to.
* @param indications the list of indications to add.
*/
private void addIndicationsToDBObject(DBObject object, List<FoundIndication> indications) {
if (indications != null && !indications.isEmpty()) {
List<DBObject> foundIndicationObjects = new LinkedList<DBObject>();
for (FoundIndication foundIndication : indications) {
DBObject foundIndicationObject = new BasicDBObject();
foundIndicationObject.put("pattern", foundIndication.getPattern());
foundIndicationObject.put("matchingFile", foundIndication.getMatchingFile());
foundIndicationObject.put("matchingString", foundIndication.getMatchingString());
foundIndicationObjects.add(foundIndicationObject);
}
object.put("indications", foundIndicationObjects);
}
}
@Override
public Descriptor<KnowledgeBase> getDescriptor() {
return Jenkins.getInstance().getDescriptorByType(MongoDBKnowledgeBaseDescriptor.class);
}
/**
* Gets the connection to the MongoDB
* @return the Mongo.
* @throws UnknownHostException if the host cannot be found.
*/
private Mongo getMongoConnection() throws UnknownHostException {
if (mongo == null) {
mongo = new Mongo(host, port);
}
return mongo;
}
/**
* Gets the DB.
* @return The DB.
* @throws UnknownHostException if the host cannot be found.
* @throws AuthenticationException if we cannot authenticate towards the database.
*/
private DB getDb() throws UnknownHostException, AuthenticationException {
if (db == null) {
db = getMongoConnection().getDB(dbName);
}
if (Util.fixEmpty(userName) != null && Util.fixEmpty(Secret.toString(password)) != null) {
char[] pwd = password.getPlainText().toCharArray();
if (!db.authenticate(userName, pwd)) {
throw new AuthenticationException("Could not athenticate with the mongo database");
}
}
return db;
}
/**
* Gets the DBCollection.
* @return The db collection.
* @throws UnknownHostException if the host cannot be found.
* @throws AuthenticationException if we cannot authenticate towards the database.
*/
private DBCollection getCollection() throws UnknownHostException, AuthenticationException {
if (collection == null) {
collection = getDb().getCollection(COLLECTION_NAME);
}
return collection;
}
/**
* Gets the Statistics DBCollection.
* @return The statistics db collection.
* @throws UnknownHostException if the host cannot be found.
* @throws AuthenticationException if we cannot authenticate towards the database.
*/
private synchronized DBCollection getStatisticsCollection() throws UnknownHostException, AuthenticationException {
if (statisticsCollection == null) {
statisticsCollection = getDb().getCollection(STATISTICS_COLLECTION_NAME);
}
return statisticsCollection;
}
/**
* Gets the JacksonDBCollection for FailureCauses.
* @return The jackson db collection.
* @throws UnknownHostException if the host cannot be found.
* @throws AuthenticationException if we cannot authenticate towards the database.
*/
private synchronized JacksonDBCollection<FailureCause, String> getJacksonCollection()
throws UnknownHostException, AuthenticationException {
if (jacksonCollection == null) {
if (collection == null) {
collection = getCollection();
}
jacksonCollection = JacksonDBCollection.wrap(collection, FailureCause.class, String.class);
}
return jacksonCollection;
}
/**
* Gets the JacksonDBCollection for Statistics.
* @return The jackson db collection.
* @throws UnknownHostException if the host cannot be found.
* @throws AuthenticationException if we cannot authenticate towards the database.
*/
private synchronized JacksonDBCollection<Statistics, String> getJacksonStatisticsCollection()
throws UnknownHostException, AuthenticationException {
if (jacksonStatisticsCollection == null) {
if (statisticsCollection == null) {
statisticsCollection = getStatisticsCollection();
}
jacksonStatisticsCollection = JacksonDBCollection.wrap(statisticsCollection, Statistics.class, String.class);
}
return jacksonStatisticsCollection;
}
/**
* Descriptor for {@link MongoDBKnowledgeBase}.
*/
@Extension
public static class MongoDBKnowledgeBaseDescriptor extends KnowledgeBaseDescriptor {
@Override
public String getDisplayName() {
return Messages.MongoDBKnowledgeBase_DisplayName();
}
/**
* Convenience method for jelly.
* @return the default port.
*/
public int getDefaultPort() {
return MONGO_DEFAULT_PORT;
}
/**
* Checks that the host name is not empty.
*
* @param value the pattern to check.
* @return {@link hudson.util.FormValidation#ok()} if everything is well.
*/
public FormValidation doCheckHost(@QueryParameter("value") final String value) {
if (Util.fixEmpty(value) == null) {
return FormValidation.error("Please provide a host name!");
} else {
Matcher m = Pattern.compile("\\s").matcher(value);
if (m.find()) {
return FormValidation.error("Host name contains white space!");
}
return FormValidation.ok();
}
}
/**
* Checks that the port number is not empty and is a number.
*
* @param value the port number to check.
* @return {@link hudson.util.FormValidation#ok()} if everything is well.
*/
public FormValidation doCheckPort(@QueryParameter("value") String value) {
try {
Long.parseLong(value);
return FormValidation.ok();
} catch (NumberFormatException e) {
return FormValidation.error("Please provide a port number!");
}
}
/**
* Checks that the database name is not empty.
*
* @param value the database name to check.
* @return {@link hudson.util.FormValidation#ok()} if everything is well.
*/
public FormValidation doCheckDBName(@QueryParameter("value") String value) {
if (value == null || value.isEmpty()) {
return FormValidation.error("Please provide a database name!");
} else {
Matcher m = Pattern.compile("\\s").matcher(value);
if (m.find()) {
return FormValidation.error("Database name contains white space!");
}
return FormValidation.ok();
}
}
/**
* Tests if the provided parameters can connect to the Mongo database.
* @param host the host name.
* @param port the port.
* @param dbName the database name.
* @param userName the user name.
* @param password the password.
* @return {@link FormValidation#ok() } if can be done,
* {@link FormValidation#error(java.lang.String) } otherwise.
*/
public FormValidation doTestConnection(
@QueryParameter("host") final String host,
@QueryParameter("port") final int port,
@QueryParameter("dbName") final String dbName,
@QueryParameter("userName") final String userName,
@QueryParameter("password") final String password) {
MongoDBKnowledgeBase base = new MongoDBKnowledgeBase(host, port, dbName, userName,
Secret.fromString(password), false, false);
try {
base.getCollection();
} catch (Exception e) {
return FormValidation.error(e, Messages.MongoDBKnowledgeBase_ConnectionError());
}
return FormValidation.ok(Messages.MongoDBKnowledgeBase_ConnectionOK());
}
}
}