/*
* NOTE: This copyright does *not* cover user programs that use Hyperic
* program services by normal system calls through the application
* program interfaces provided as part of the Hyperic Plug-in Development
* Kit or the Hyperic Client Development Kit - this is merely considered
* normal use of the program, and does *not* fall under the heading of
* "derived work".
*
* Copyright (C) [2004-2011], VMware, Inc.
* This file is part of Hyperic.
*
* Hyperic is free software; you can redistribute it and/or modify
* it under the terms version 2 of the GNU General Public License as
* published by the Free Software Foundation. This program is distributed
* in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
* even the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*/
package org.hyperic.hq.measurement.server.session;
import static org.hyperic.hq.measurement.MeasurementConstants.AVAIL_DOWN;
import static org.hyperic.hq.measurement.MeasurementConstants.AVAIL_NULL;
import static org.hyperic.hq.measurement.MeasurementConstants.AVAIL_PAUSED;
import static org.hyperic.hq.measurement.MeasurementConstants.AVAIL_UNKNOWN;
import static org.hyperic.hq.measurement.MeasurementConstants.AVAIL_UP;
import java.math.BigDecimal;
import java.math.MathContext;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import javax.annotation.PostConstruct;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hibernate.ObjectNotFoundException;
import org.hyperic.hq.appdef.Agent;
import org.hyperic.hq.appdef.server.session.AgentDAO;
import org.hyperic.hq.appdef.server.session.Platform;
import org.hyperic.hq.appdef.shared.AppdefEntityConstants;
import org.hyperic.hq.appdef.shared.AppdefEntityID;
import org.hyperic.hq.appdef.shared.AppdefEntityValue;
import org.hyperic.hq.appdef.shared.AppdefResourceValue;
import org.hyperic.hq.appdef.shared.AppdefUtil;
import org.hyperic.hq.authz.server.session.AuthzSubject;
import org.hyperic.hq.authz.server.session.Resource;
import org.hyperic.hq.authz.server.session.ResourceGroup;
import org.hyperic.hq.authz.shared.AuthzConstants;
import org.hyperic.hq.authz.shared.AuthzSubjectManager;
import org.hyperic.hq.authz.shared.PermissionException;
import org.hyperic.hq.authz.shared.PermissionManagerFactory;
import org.hyperic.hq.authz.shared.ResourceGroupManager;
import org.hyperic.hq.authz.shared.ResourceManager;
import org.hyperic.hq.common.SystemException;
import org.hyperic.hq.common.util.MessagePublisher;
import org.hyperic.hq.events.EventConstants;
import org.hyperic.hq.events.MaintenanceEvent;
import org.hyperic.hq.events.ext.RegisteredTriggers;
import org.hyperic.hq.measurement.MeasurementConstants;
import org.hyperic.hq.measurement.MeasurementNotFoundException;
import org.hyperic.hq.measurement.TimingVoodoo;
import org.hyperic.hq.measurement.ext.DownMetricValue;
import org.hyperic.hq.measurement.ext.MeasurementEvent;
import org.hyperic.hq.measurement.shared.AvailabilityManager;
import org.hyperic.hq.measurement.shared.HighLowMetricValue;
import org.hyperic.hq.measurement.shared.MeasurementManager;
import org.hyperic.hq.product.AvailabilityMetricValue;
import org.hyperic.hq.product.MetricValue;
import org.hyperic.hq.product.PlatformDetector;
import org.hyperic.hq.stats.ConcurrentStatsCollector;
import org.hyperic.hq.zevents.ZeventManager;
import org.hyperic.util.pager.PageControl;
import org.hyperic.util.pager.PageList;
import org.hyperic.util.timer.StopWatch;
import org.quartz.SchedulerException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
/**
* The AvailabityManagerImpl class is a stateless session bean that can be used
* to retrieve Availability Data RLE points
*
*/
@Service
@Transactional
public class AvailabilityManagerImpl implements AvailabilityManager {
private final Log log = LogFactory.getLog(AvailabilityManagerImpl.class);
private final Log traceLog = LogFactory.getLog(AvailabilityManagerImpl.class.getName() + "Trace");
// private static final double AVAIL_NULL = MeasurementConstants.AVAIL_NULL;
// private static final double AVAIL_DOWN = MeasurementConstants.AVAIL_DOWN;
// private static final double AVAIL_UNKNOWN = MeasurementConstants.AVAIL_UNKNOWN;
private static final int IND_MIN = MeasurementConstants.IND_MIN;
private static final int IND_AVG = MeasurementConstants.IND_AVG;
private static final int IND_MAX = MeasurementConstants.IND_MAX;
private static final int IND_CFG_COUNT = MeasurementConstants.IND_CFG_COUNT;
private static final int IND_LAST_TIME = MeasurementConstants.IND_LAST_TIME;
private static final int IND_UP_TIME = IND_LAST_TIME + 1;
private static final int IND_TOTAL_TIME = IND_UP_TIME + 1;
private static final long MAX_AVAIL_TIMESTAMP = AvailabilityDataRLE.getLastTimestamp();
private static final String ALL_EVENTS_INTERESTING_PROP = "org.hq.triggers.all.events.interesting";
private static final int DEFAULT_INTERVAL = 60;
private static final String AVAIL_MANAGER_METRICS_INSERTED = ConcurrentStatsCollector.AVAIL_MANAGER_METRICS_INSERTED;
private static final long MAX_DATA_BACKLOG_TIME = 7 * MeasurementConstants.DAY;
private AuthzSubjectManager authzSubjectManager;
private MeasurementManager measurementManager;
private ResourceGroupManager groupManager;
private ResourceManager resourceManager;
private MessagePublisher messenger;
private AvailabilityDataDAO availabilityDataDAO;
private AvailabilityFallbackCheckQue fallbackCheckQue;
private MeasurementDAO measurementDAO;
private MessagePublisher messagePublisher;
private RegisteredTriggers registeredTriggers;
private AvailabilityCache availabilityCache;
private ConcurrentStatsCollector concurrentStatsCollector;
private AgentDAO agentDAO;
@Autowired
public AvailabilityManagerImpl(AuthzSubjectManager authzSubjectManager, ResourceManager resourceManager,
ResourceGroupManager groupManager, MessagePublisher messenger, AvailabilityDataDAO availabilityDataDAO,
MeasurementDAO measurementDAO, MessagePublisher messagePublisher, RegisteredTriggers registeredTriggers,
AvailabilityCache availabilityCache, AgentDAO agentDAO, ConcurrentStatsCollector concurrentStatsCollector) {
this.authzSubjectManager = authzSubjectManager;
this.resourceManager = resourceManager;
this.groupManager = groupManager;
this.messenger = messenger;
this.availabilityDataDAO = availabilityDataDAO;
this.measurementDAO = measurementDAO;
this.messagePublisher = messagePublisher;
this.registeredTriggers = registeredTriggers;
this.availabilityCache = availabilityCache;
this.concurrentStatsCollector = concurrentStatsCollector;
this.agentDAO = agentDAO;
//this.measurementIDsMonitoredByServer = new HashSet<Integer>();
this.fallbackCheckQue = new AvailabilityFallbackCheckQue();
}
@PostConstruct
public void initStatsCollector() {
concurrentStatsCollector.register(ConcurrentStatsCollector.AVAIL_MANAGER_METRICS_INSERTED);
}
// To break AvailabilityManager - MeasurementManager circular dependency
// TODO: Check why we need this when we have
// MeasurementManagerImpl.setCircularDependencies()?
@Autowired
public void setMeasurementManager(MeasurementManager measurementManager) {
this.measurementManager = measurementManager;
}
/**
*
*/
@Transactional(readOnly = true)
public Measurement getAvailMeasurement(Resource resource) {
return measurementDAO.findAvailMeasurement(resource);
}
/**
*
*/
@Transactional(readOnly = true)
public List<Measurement> getPlatformResources() {
return measurementDAO.findAvailMeasurementsByInstances(AppdefEntityConstants.APPDEF_TYPE_PLATFORM, null);
}
public AvailabilityFallbackCheckQue getFallbackCheckQue() {
return this.fallbackCheckQue;
}
/**
* @return Down time in ms for the Resource availability
*
*
*/
@Transactional(readOnly = true)
public long getDowntime(Resource resource, long rangeBegin, long rangeEnd) throws MeasurementNotFoundException {
Measurement meas = measurementDAO.findAvailMeasurement(resource);
if (meas == null) {
throw new MeasurementNotFoundException("Availability measurement " + "not found for resource "
+ resource.getId());
}
List<AvailabilityDataRLE> availInfo = availabilityDataDAO
.getHistoricalAvails(meas, rangeBegin, rangeEnd, false);
long rtn = 0l;
for (AvailabilityDataRLE avail : availInfo) {
if (avail.getAvailVal() != AVAIL_DOWN) {
continue;
}
long dataDownEndTime = avail.getEndtime();
if (dataDownEndTime == MAX_AVAIL_TIMESTAMP) {
dataDownEndTime = System.currentTimeMillis();
}
if (dataDownEndTime > rangeEnd) {
// use range end if the data down end time is greater than the
// end range.
dataDownEndTime = rangeEnd;
}
long dataDownStartTime = avail.getStartime();
// Make sure the start of the down time is not earlier then the
// begin time
if (dataDownStartTime < rangeBegin) {
dataDownStartTime = rangeBegin;
}
rtn += (dataDownEndTime - dataDownStartTime);
}
return rtn;
}
/**
* @return List of all measurement ids for availability, ordered
*
*
*/
@Transactional(readOnly = true)
public List<Integer> getAllAvailIds() {
return measurementDAO.findAllAvailIds();
}
/**
*
*/
@Transactional(readOnly = true)
public List<Measurement> getAvailMeasurementChildren(Resource resource, String resourceRelationType) {
final List<Integer> sList = Collections.singletonList(resource.getId());
List<Measurement> rtn = getAvailMeasurementChildren(sList, resourceRelationType).get(resource.getId());
if (rtn == null) {
rtn = new ArrayList<Measurement>(0);
}
return rtn;
}
/**
* @param {@link List} of {@link Integer} resource ids
* @return {@link Map} of {@link Integer} to {@link List} of
* {@link Measurement}
*
*/
@Transactional(readOnly = true)
public Map<Integer, List<Measurement>> getAvailMeasurementChildren(List<Integer> resourceIds,
String resourceRelationType) {
final List<Object[]> objects = measurementDAO.findRelatedAvailMeasurements(resourceIds, resourceRelationType);
return convertAvailMeasurementListToMap(objects);
}
/**
*
*/
@Transactional(readOnly = true)
public List<Measurement> getAvailMeasurementParent(Resource resource, String resourceRelationType) {
final List<Integer> sList = Collections.singletonList(resource.getId());
List<Measurement> rtn = getAvailMeasurementParent(sList, resourceRelationType).get(resource.getId());
if (rtn == null) {
rtn = new ArrayList<Measurement>(0);
}
return rtn;
}
/**
*
*/
@Transactional(readOnly = true)
public Map<Integer, List<Measurement>> getAvailMeasurementParent(List<Integer> resourceIds,
String resourceRelationType) {
final List<Object[]> objects = measurementDAO.findParentAvailMeasurements(resourceIds, resourceRelationType);
return convertAvailMeasurementListToMap(objects);
}
/**
*
*/
@Transactional(readOnly = true)
public Map<Integer, List<Measurement>> getAvailMeasurementDirectParent(List<Integer> resourceIds,
String resourceRelationType) {
final List<Object[]> objects = measurementDAO.findDirectParentAvailMeasurements(resourceIds, resourceRelationType);
return convertAvailMeasurementListToMap(objects);
}
private Map<Integer, List<Measurement>> convertAvailMeasurementListToMap(List<Object[]> objects) {
final Map<Integer, List<Measurement>> rtn = new HashMap<Integer, List<Measurement>>(objects.size());
for (Object[] o : objects) {
final Integer rId = (Integer) o[0];
final Measurement m = (Measurement) o[1];
List<Measurement> tmp;
if (null == (tmp = rtn.get(rId))) {
tmp = new ArrayList<Measurement>();
rtn.put(rId, tmp);
}
tmp.add(m);
}
return rtn;
}
/**
* Get Availability measurements (disabled) in scheduled downtime.
*/
@Transactional(readOnly = true)
public Map<Integer, Measurement> getAvailMeasurementsInDowntime(Collection<AppdefEntityID> eids) {
Map<Integer, Measurement> measMap = new HashMap<Integer, Measurement>();
try {
AuthzSubject overlord = authzSubjectManager.getOverlordPojo();
// TODO: Resolve circular dependency and autowire
// MaintenanceEventManager
List<MaintenanceEvent> events = PermissionManagerFactory.getInstance().getMaintenanceEventManager()
.getMaintenanceEvents(overlord, MaintenanceEvent.STATE_RUNNING);
for (MaintenanceEvent event : events) {
AppdefEntityID entityId = event.getAppdefEntityID();
Collection<Resource> resources = null;
if (entityId.isGroup()) {
ResourceGroup group = groupManager.findResourceGroupById(entityId.getId());
resources = groupManager.getMembers(group);
} else {
Resource resource = resourceManager.findResource(entityId);
resources = Collections.singletonList(resource);
}
for (Resource resource : resources) {
List<Measurement> measurements = getAvailMeasurementChildren(resource,
AuthzConstants.ResourceEdgeContainmentRelation);
measurements.add(getAvailMeasurement(resource));
if (!measurements.isEmpty()) {
for (Measurement m : measurements) {
if (m == null) {
// measurement could be null if resource has not
// been configured
continue;
}
Resource r = m.getResource();
if (r == null || r.isInAsyncDeleteState()) {
continue;
}
// availability measurement in scheduled downtime
// are disabled
if (!m.isEnabled() && eids.contains(AppdefUtil.newAppdefEntityId(r))) {
measMap.put(r.getId(), m);
}
}
}
}
}
} catch (Exception e) {
log.error("Could not find availability measurements in downtime: " + e.getMessage(), e);
}
return measMap;
}
/**
* TODO: Can this method be combined with the one that takes an array?
*
*
*/
@Transactional(readOnly = true)
public PageList<HighLowMetricValue> getHistoricalAvailData(Measurement m, long begin, long end, PageControl pc,
boolean prependUnknowns) {
List<AvailabilityDataRLE> availInfo = availabilityDataDAO.getHistoricalAvails(m, begin, end, pc.isDescending());
return getPageList(availInfo, begin, end, m.getInterval(), prependUnknowns);
}
/**
* Fetches historical availability encapsulating the specified time range
* for each measurement id in mids;
*
* @param mids
* measurement ids
* @param begin
* time range start
* @param end
* time range end
* @param interval
* interval of each time range window
* @param pc
* page control
* @param prependUnknowns
* determines whether to prepend AVAIL_UNKNOWN if the
* corresponding time window is not accounted for in the
* database. Since availability is contiguous this will not occur
* unless the time range precedes the first availability point.
* @see org.hyperic.hq.measurement.MeasurementConstants#AVAIL_UNKNOWN
*
*/
@Transactional(readOnly = true)
public PageList<HighLowMetricValue> getHistoricalAvailData(Integer[] mids, long begin, long end, long interval,
PageControl pc, boolean prependUnknowns) {
if (mids.length == 0) {
return new PageList<HighLowMetricValue>();
}
List<AvailabilityDataRLE> availInfo = availabilityDataDAO.getHistoricalAvails(mids, begin, end,
pc.isDescending());
return getPageList(availInfo, begin, end, interval, prependUnknowns);
}
/**
* Get the list of Raw RLE objects for a resource
*
* @return List<AvailabilityDataRLE>
*
*/
@Transactional(readOnly = true)
public List<AvailabilityDataRLE> getHistoricalAvailData(Resource res, long begin, long end) {
return availabilityDataDAO.getHistoricalAvails(res, begin, end);
}
private Collection<HighLowMetricValue> getDefaultHistoricalAvail(long timestamp) {
HighLowMetricValue[] rtn = new HighLowMetricValue[DEFAULT_INTERVAL];
Arrays.fill(rtn, new HighLowMetricValue(AVAIL_UNKNOWN, timestamp));
return Arrays.asList(rtn);
}
private PageList<HighLowMetricValue> getPageList(List<AvailabilityDataRLE> availInfo, long begin, long end,
long interval, boolean prependUnknowns) {
PageList<HighLowMetricValue> rtn = new PageList<HighLowMetricValue>();
for (Iterator<AvailabilityDataRLE> it = availInfo.iterator(); it.hasNext();) {
AvailabilityDataRLE rle = it.next();
long availStartime = rle.getStartime();
long availEndtime = rle.getEndtime();
//skip measurements that are before first time slot
if (availEndtime < begin) {
continue;
}
LinkedList<AvailabilityDataRLE> queue = new LinkedList<AvailabilityDataRLE>();
queue.add(rle);
int i = 0;
for (long curr = begin; curr <= end; curr += interval) {
long next = curr + interval;
next = (next > end) ? end : next;
long endtime = ((AvailabilityDataRLE) queue.getFirst()).getEndtime();
//while next time slot is after measurement time range
while (next > endtime) {
// it should not be the case that there are no more
// avails in the array, but we need to handle it
if (it.hasNext()) {
AvailabilityDataRLE tmp = (AvailabilityDataRLE) it.next();
queue.addFirst(tmp);
endtime = tmp.getEndtime();
} else {
endtime = availEndtime;
int measId = rle.getMeasurement().getId().intValue();
String msg = "Measurement, " + measId + ", for interval " + begin + " - " + end
+ " did not return a value for range " + curr + " - " + (curr + interval);
log.warn(msg);
}
}
endtime = availEndtime;
while (curr > endtime) {
queue.removeLast();
// this should not happen unless the above !it.hasNext()
// else condition is true
if (queue.size() == 0) {
rle = new AvailabilityDataRLE(rle.getMeasurement(), rle.getEndtime(), next, AVAIL_UNKNOWN);
queue.addLast(rle);
}
rle = (AvailabilityDataRLE) queue.getLast();
availStartime = rle.getStartime();
availEndtime = rle.getEndtime();
endtime = availEndtime;
}
HighLowMetricValue val;
if (curr >= availStartime) {
val = getMetricValue(queue, curr);
} else if (prependUnknowns) {
val = new HighLowMetricValue(AVAIL_UNKNOWN, curr);
val.incrementCount();
} else {
i++;
continue;
}
if (rtn.size() <= i) {
rtn.add(round(val));
} else {
updateMetricValue(val, (HighLowMetricValue) rtn.get(i));
}
i++;
}
}
if (rtn.size() == 0) {
rtn.addAll(getDefaultHistoricalAvail(end));
}
return rtn;
}
private HighLowMetricValue round(HighLowMetricValue val) {
final BigDecimal b = new BigDecimal(val.getValue(), new MathContext(10));
val.setValue(b.doubleValue());
return val;
}
private HighLowMetricValue updateMetricValue(HighLowMetricValue newVal, HighLowMetricValue oldVal) {
if (newVal.getHighValue() == AVAIL_UNKNOWN || newVal.getHighValue() > oldVal.getHighValue()) {
oldVal.setHighValue(newVal.getHighValue());
}
if (newVal.getLowValue() == AVAIL_UNKNOWN || newVal.getLowValue() < oldVal.getLowValue()) {
oldVal.setLowValue(newVal.getLowValue());
}
int count = oldVal.getCount();
if (oldVal.getValue() == AVAIL_UNKNOWN) {
double value = newVal.getValue();
oldVal.setValue(value);
oldVal.setCount(1);
} else if (newVal.getValue() == AVAIL_UNKNOWN) {
return oldVal;
} else {
double value = ((newVal.getValue() + (oldVal.getValue() * count))) / (count + 1);
oldVal.setValue(value);
oldVal.incrementCount();
round(oldVal);
}
return oldVal;
}
private HighLowMetricValue getMetricValue(List<AvailabilityDataRLE> avails, long timestamp) {
HighLowMetricValue val;
if (avails.size() == 1) {
AvailabilityDataRLE rle = avails.get(0);
val = new HighLowMetricValue(rle.getAvailVal(), timestamp);
} else if (avails.size() == 2) {
AvailabilityDataRLE rle = null;
// HHQ-5244
// If there are only two RLEs then averaging them doesn't make sense
// as it can make
// a value of 0 show up as .5 for an interval
// The following logic makes sure that the actual value is provided
if (avails.get(0).getEndtime() < timestamp && avails.get(1).getStartime() >= timestamp) {
rle = avails.get(0);
} else {
rle = avails.get(1);
}
val = new HighLowMetricValue(rle.getAvailVal(), timestamp);
}
else {
double value = 0;
for (AvailabilityDataRLE rle : avails) {
double availVal = rle.getAvailVal();
value += availVal;
}
value = value / avails.size();
val = new HighLowMetricValue(value, timestamp);
}
val.incrementCount();
return val;
}
/**
* @return {@link Map} of {@link Measurement} to {@link double[]}. Array is
* comprised of 5 elements: [IND_MIN] [IND_AVG] [IND_MAX]
* [IND_CFG_COUNT] [IND_LAST_TIME]
*
*/
@Transactional(readOnly = true)
public Map<Integer, double[]> getAggregateData(Integer[] mids, long begin, long end) {
List<Object[]> avails = availabilityDataDAO.findAggregateAvailability(mids, begin, end);
return getAggData(avails, false);
}
/**
* @return {@link Map} of {@link MeasurementTemplate.getId} to {@link
* double[]}. Array is comprised of 5 elements: [IND_MIN] [IND_AVG]
* [IND_MAX] [IND_CFG_COUNT] [IND_LAST_TIME]
*
*/
@Transactional(readOnly = true)
public Map<Integer, double[]> getAggregateDataByTemplate(Integer[] mids, long begin, long end) {
List<Object[]> avails = availabilityDataDAO.findAggregateAvailability(mids, begin, end);
return getAggData(avails, true);
}
@Transactional(readOnly = true)
public Map<Integer, double[]> getAggregateDataAndAvailUpByMetric(final List<Integer> mids, long begin, long end)
throws SQLException {
Map<Integer,Double> avails = availabilityDataDAO.findAggregateAvailabilityUp(mids, begin, end);
Map<Integer, double[]> msmtToAvg = new HashMap<Integer, double[]>();
Set<Integer> unavailMsmtsIds = new HashSet<Integer>(mids);
if (null != avails) {
for (Map.Entry<Integer, Double> availIdToAvg : avails.entrySet()) {
Integer availId = availIdToAvg.getKey();
if (!unavailMsmtsIds.remove(availId)) {
throw new RuntimeException("unknown availability measurement returned while querying for availability data");
}
Double availAvg = availIdToAvg.getValue();
double[] data = new double[IND_TOTAL_TIME + 1];
data[IND_AVG] = availAvg.doubleValue();
msmtToAvg.put(availId, data);
}
}
// resources which were not up in the time frame, wouldn't be returned, and have to be initialized
final double[] availDownData = new double[IND_TOTAL_TIME + 1];
for (Integer unavailMsmtId : unavailMsmtsIds) {
msmtToAvg.put(unavailMsmtId, availDownData);
}
return msmtToAvg;
}
/**
* @return {@link Map} of {@link MeasurementTemplate.getId} to {@link
* double[]}. Array is comprised of 5 elements: [IND_MIN] [IND_AVG]
* [IND_MAX] [IND_CFG_COUNT] [IND_LAST_TIME]
*
*/
@Transactional(readOnly = true)
public Map<Integer, double[]> getAggregateData(Integer[] tids, Integer[] iids, long begin, long end) {
List<Object[]> avails = availabilityDataDAO.findAggregateAvailability(tids, iids, begin, end);
return getAggData(avails, true);
}
private Map<Integer, double[]> getAggData(List<Object[]> avails, boolean useTidKey) {
Map<Integer, double[]> rtn = new HashMap<Integer, double[]>();
if (avails.size() == 0) {
// Nothing to do, return an empty Map.
return rtn;
}
for (Object[] objs : avails) {
double[] data;
Integer key = null;
if (useTidKey) {
if (objs[0] instanceof Measurement) {
key = ((Measurement) objs[0]).getTemplate().getId();
} else {
key = (Integer) objs[0];
}
} else {
key = ((Measurement) objs[0]).getId();
}
if (null == (data = (double[]) rtn.get(key))) {
data = new double[IND_TOTAL_TIME + 1];
data[IND_MIN] = AVAIL_UP;
data[IND_MAX] = AVAIL_PAUSED;
rtn.put(key, data);
}
data[IND_MIN] = Math.min(data[IND_MIN], ((Double) objs[1]).doubleValue());
data[IND_MAX] = Math.max(data[IND_MAX], ((Double) objs[3]).doubleValue());
// Expect data to be sorted by end time, so that the last value
// returned is the final count and the last value
data[IND_CFG_COUNT] += (objs[4] == null) ? 0 : ((java.lang.Number) objs[4]).doubleValue();
data[IND_LAST_TIME] = ((Double) objs[2]).doubleValue();
data[IND_UP_TIME] += ((Double) objs[5]).doubleValue();
data[IND_TOTAL_TIME] += ((Double) objs[6]).doubleValue();
}
// Now calculate the average value
for (double[] data : rtn.values()) {
data[IND_AVG] += data[IND_UP_TIME] / data[IND_TOTAL_TIME];
}
return rtn;
}
/**
* @param resources
* Collection may be of type {@link Resource},
* {@link AppdefEntityId}, {@link AppdefEntityValue},
* {@link AppdefResourceValue} or {@link Integer}
* @param measCache
* Map<Integer, List> optional arg (may be null) to supply
* measurement id(s) of ResourceIds. Integer => Resource.getId().
* If a measurement is not specified in the measCache parameter
* it will be added to the map
* @return Map<Integer, MetricValue> Integer => Measurement.getId()
*
*/
@Transactional(readOnly = true)
public Map<Integer, MetricValue> getLastAvail(Collection<? extends Object> resources,
Map<Integer, List<Measurement>> measCache) {
final Set<Integer> midsToGet = new HashSet<Integer>(resources.size());
final List<Resource> resToGet = new ArrayList<Resource>(resources.size());
for (Object o : resources) {
Resource resource = null;
if (o instanceof AppdefEntityValue) {
AppdefEntityValue rv = (AppdefEntityValue) o;
AppdefEntityID aeid = rv.getID();
resource = resourceManager.findResource(aeid);
} else if (o instanceof AppdefEntityID) {
AppdefEntityID aeid = (AppdefEntityID) o;
resource = resourceManager.findResource(aeid);
} else if (o instanceof Resource) {
resource = (Resource) o;
} else if (o instanceof AppdefResourceValue) {
AppdefResourceValue res = (AppdefResourceValue) o;
AppdefEntityID aeid = res.getEntityId();
resource = resourceManager.findResource(aeid);
} else {
resource = resourceManager.findResourceById((Integer) o);
}
List<Measurement> measurements = null;
try {
if (resource == null || resource.isInAsyncDeleteState()) {
continue;
}
} catch (ObjectNotFoundException e) {
// resource is in async delete state, ignore
log.debug("resource not found from object=" + o, e);
continue;
}
if (measCache != null) {
measurements = measCache.get(resource.getId());
}
if (measurements == null || measurements.size() == 0) {
resToGet.add(resource);
continue;
}
for (Measurement m : measurements) {
// populate the Map if value doesn't exist
if (measCache != null) {
List<Measurement> measids = measCache.get(m.getResource().getId());
if (measids == null) {
measids = new ArrayList<Measurement>();
measids.add(m);
measCache.put(m.getResource().getId(), measids);
}
}
midsToGet.add(m.getId());
}
}
if (!resToGet.isEmpty()) {
final Collection<List<Measurement>> measIds = measurementManager.getAvailMeasurements(resToGet).values();
for (List<Measurement> measurementList : measIds) {
for (Measurement m : measurementList) {
midsToGet.add(m.getId());
}
}
}
return getLastAvail((Integer[]) midsToGet.toArray(new Integer[0]));
}
/**
*
*/
@Transactional(readOnly = true)
public MetricValue getLastAvail(Measurement m) {
Map<Integer, MetricValue> map = getLastAvail(new Integer[] { m.getId() });
MetricValue mv = (MetricValue) map.get(m.getId());
if (mv == null) {
return new MetricValue(AVAIL_UNKNOWN, System.currentTimeMillis());
} else {
return mv;
}
}
/**
* Only unique measurement ids should be passed in. Duplicate measurement
* ids will be filtered out from the returned Map if present.
*
* @return {@link Map} of {@link Integer} to {@link MetricValue} Integer is
* the measurementId
*
*/
@Transactional(readOnly = true)
public Map<Integer, MetricValue> getLastAvail(Integer[] mids) {
if (mids.length == 0) {
return Collections.emptyMap();
}
// Don't modify callers array
final List<Integer> midList = Collections.unmodifiableList(Arrays.asList(mids));
final Map<Integer, MetricValue> rtn = new HashMap<Integer, MetricValue>(midList.size());
final List<AvailabilityDataRLE> list = availabilityDataDAO.findLastAvail(midList);
for (AvailabilityDataRLE avail : list) {
final Integer mid = avail.getMeasurement().getId();
final AvailabilityMetricValue mVal = new AvailabilityMetricValue(avail.getAvailVal(), avail.getStartime(),
avail.getApproxEndtime());
rtn.put(mid, mVal);
}
// fill in missing measurements
final long now = TimingVoodoo.roundDownTime(System.currentTimeMillis(), MeasurementConstants.MINUTE);
for (Integer mid : midList) {
if (!rtn.containsKey(mid)) {
final MetricValue mVal = new MetricValue(AVAIL_UNKNOWN, now);
rtn.put(mid, mVal);
}
}
return rtn;
}
/**
* @return {@link Map} of {@link Integer} representing {@link Resource}Id to {@link DownMetricValue}
*/
@Transactional(readOnly = true)
public Map<Integer, DownMetricValue> getUnavailResMap() {
final Map<Integer, DownMetricValue> rtn = new HashMap<Integer, DownMetricValue>();
final List<AvailabilityDataRLE> unavails = availabilityDataDAO.getDownMeasurements(null);
for (final AvailabilityDataRLE rle : unavails) {
final Measurement meas = rle.getMeasurement();
final Resource resource = meas.getResource();
if (resource == null || resource.isInAsyncDeleteState()) {
continue;
}
final long timestamp = rle.getStartime();
final Integer mid = meas.getId();
final MetricValue val = new MetricValue(AVAIL_DOWN, timestamp);
rtn.put(resource.getId(), new DownMetricValue(meas.getEntityId(), mid, val));
}
return rtn;
}
/**
* @param includes List<Integer> of mids. If includes is null then all
* unavail entities will be returned.
*
*/
@Transactional(readOnly = true)
public List<DownMetricValue> getUnavailEntities(List<Integer> includes) {
List<DownMetricValue> rtn;
if (includes != null) {
rtn = new ArrayList<DownMetricValue>(includes.size());
} else {
rtn = new ArrayList<DownMetricValue>();
}
List<AvailabilityDataRLE> unavails = availabilityDataDAO.getDownMeasurements(includes);
for (AvailabilityDataRLE rle : unavails) {
Measurement meas = rle.getMeasurement();
long timestamp = rle.getStartime();
Integer mid = meas.getId();
MetricValue val = new MetricValue(AVAIL_DOWN, timestamp);
rtn.add(new DownMetricValue(meas.getEntityId(), mid, val));
}
return rtn;
}
/**
* Add a single Availablility Data point.
*
* @mid The Measurement id
* @mval The MetricValue to store.
*
*/
public void addData(Integer mid, MetricValue mval) {
List<DataPoint> l = new ArrayList<DataPoint>(1);
l.add(new DataPoint(mid, mval));
addData(l);
}
/**
* Process Availability data. The default behavior is to send the data
* points to the event handlers.
*
* @param availPoints
* List of DataPoints
*
*
*/
public void addData(List<DataPoint> availPoints) {
addData(availPoints, true, false);
}
public void addData(List<DataPoint> availDataPoints, boolean sendData) {
addData(availDataPoints, sendData, false);
}
/**
* Process Availability data.
* For each measurement Id (for each resource's availability):
* If Availability Value is the same as in the DB (AvailabilityDataRLE), only its timestamp is updated in the availabilityCache.
* Otherwise (availability status changed) - update the availabilityCache and the DB record.
* @param availPoints List of DataPoints
* @param sendData Indicates whether to send the data to event handlers. The
* default behavior is true. If false, the calling method should call
* sendDataToEventHandlers directly afterwards.
*
* @param availPoints
* List of DataPoints
* @param sendData
* Indicates whether to send the data to event handlers. The
* default behavior is true. If false, the calling method should
* call sendDataToEventHandlers directly afterwards.
*
*
*/
public void addData(List<DataPoint> availDataPoints, boolean sendData, boolean addedByServer) {
if (availDataPoints == null || availDataPoints.size() == 0) {
return;
}
Collection<DataPoint> pointsToUpdate = this.fallbackCheckQue.beforeDataUpdate(availDataPoints, addedByServer);
List<DataPoint> availPoints = new ArrayList<DataPoint>(pointsToUpdate);
List<DataPoint> updateList = new ArrayList<DataPoint>(availPoints.size());
List<DataPoint> outOfOrderAvail = new ArrayList<DataPoint>(availPoints.size());
Map<DataPoint, AvailabilityDataRLE> createMap = new HashMap<DataPoint, AvailabilityDataRLE>();
Map<DataPoint, AvailabilityDataRLE> removeMap = new HashMap<DataPoint, AvailabilityDataRLE>();
Map<Integer, StringBuilder> state = null;
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails = Collections.emptyMap();
if (log.isDebugEnabled()) {
log.debug(availDataPoints);
}
synchronized (availabilityCache) {
try {
availabilityCache.beginTran();
updateCache(availPoints, updateList, outOfOrderAvail);
currAvails = createCurrAvails(outOfOrderAvail, updateList); // get current DB Availability state for the measurements.
state = captureCurrAvailState(currAvails); // this method is called for logging.
updateStates(updateList, currAvails, createMap, removeMap);
updateOutOfOrderState(outOfOrderAvail, currAvails, createMap, removeMap);
flushCreateAndRemoves(createMap, removeMap);
checkAvailabilityState(availPoints);
logErrorInfo(state, availPoints, currAvails);
availabilityCache.commitTran();
} catch (Throwable e) {
logErrorInfo(state, availPoints, currAvails);
log.error(e.getMessage(), e);
availabilityCache.rollbackTran();
throw new SystemException(e);
}
}
concurrentStatsCollector.addStat(availPoints.size(), AVAIL_MANAGER_METRICS_INSERTED);
if (sendData) {
sendDataToEventHandlers(availPoints);
}
}
private void flushCreateAndRemoves(Map<DataPoint, AvailabilityDataRLE> createMap,
Map<DataPoint, AvailabilityDataRLE> removeMap) {
final StopWatch watch = new StopWatch();
final boolean debug = log.isDebugEnabled();
if (debug)
watch.markTimeBegin("remove");
for (Map.Entry<DataPoint, AvailabilityDataRLE> entry : removeMap.entrySet()) {
AvailabilityDataRLE rle = (AvailabilityDataRLE) entry.getValue();
// if we call remove() on an object which is already in the session
// hibernate will throw NonUniqueObjectExceptions
AvailabilityDataRLE tmp = availabilityDataDAO.get(rle.getAvailabilityDataId());
if (tmp != null) {
availabilityDataDAO.remove(tmp);
} else {
availabilityDataDAO.remove(rle);
}
}
if (debug) {
watch.markTimeEnd("remove");
watch.markTimeBegin("flush");
}
// addData() could be overwriting RLE data points (i.e. from 0.0 to 1.0)
// with the same ID. If this is the scenario, then we must run
// flush() in order to ensure that these old objects are not in the
// session when the equivalent create() on the same ID is run,
// thus avoiding NonUniqueObjectExceptions
availabilityDataDAO.getSession().flush();
if (debug) {
watch.markTimeEnd("flush");
watch.markTimeBegin("create");
}
List<MeasurementZevent> events = new ArrayList<MeasurementZevent>(createMap.entrySet().size());
for (Map.Entry<DataPoint, AvailabilityDataRLE> entry : createMap.entrySet()) {
DataPoint dp = entry.getKey();
AvailabilityDataRLE rle = (AvailabilityDataRLE) entry.getValue();
AvailabilityDataId id = new AvailabilityDataId();
Measurement m = rle.getMeasurement();
id.setMeasurement(m);
id.setStartime(rle.getStartime());
availabilityDataDAO.create(rle.getMeasurement(), rle.getStartime(), rle.getEndtime(), rle.getAvailVal());
log.debug("added: Availability "+rle.getAvailVal() + " starttime " + rle.getStartime() + " endtime " + rle.getEndtime());
events.add(new MeasurementZevent(m.getId().intValue(), dp.getMetricValue()));
}
ZeventManager.getInstance().enqueueEventsAfterCommit(events);
if (debug) {
watch.markTimeEnd("create");
log.debug("AvailabilityInserter flushCreateAndRemoves: " + watch + ", points {remove=" + removeMap.size()
+ ", create=" + createMap.size() + "}");
}
}
private void logErrorInfo(final Map<Integer, StringBuilder> oldState, final List<DataPoint> availPoints,
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails) {
if (!traceLog.isDebugEnabled()) {
return;
}
Integer mid;
Map<Integer, StringBuilder> currState = captureCurrAvailState(currAvails);
if (null != (mid = isAvailDataRLEValid(currAvails))) {
logAvailState(oldState, mid);
logStates(availPoints, mid);
logAvailState(currState, mid);
} else {
traceLog.debug("RLE Data is valid");
}
}
@SuppressWarnings("unchecked")
/**
* get AvailabilityDataRLEs for the given DataPoints' Measurement IDs, with endData within the last 7 days.
* If several AvailabilityDataRLEs exist for the same Measurement, they are listed in ascending order.
* @param outOfOrderAvail
* @param updateList
* @return
*/
private Map<Integer, TreeSet<AvailabilityDataRLE>> createCurrAvails(final List<DataPoint> outOfOrderAvail,
final List<DataPoint> updateList) {
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails = null;
final StopWatch watch = new StopWatch();
try {
if (outOfOrderAvail.size() == 0 && updateList.size() == 0) {
currAvails = Collections.EMPTY_MAP;
}
long now = TimingVoodoo.roundDownTime(System.currentTimeMillis(), 60000);
HashSet<Integer> mids = getMidsWithinAllowedDataWindow(updateList, now);
mids.addAll(getMidsWithinAllowedDataWindow(outOfOrderAvail, now));
if (mids.size() <= 0) {
currAvails = Collections.EMPTY_MAP;
}
Integer[] mIds = (Integer[]) mids.toArray(new Integer[0]);
currAvails = availabilityDataDAO.getHistoricalAvailMap(mIds, now - MAX_DATA_BACKLOG_TIME, false);
return currAvails;
} finally {
if (log.isDebugEnabled()) {
log.debug("AvailabilityInserter setCurrAvails: " + watch + ", size=" + currAvails.size());
}
}
}
/**
* allow only data given in the last MAX_DATA_BACKLOG_TIME ms
* @param states - metric data list.
* @param now - "current" timestamp
* @return set of measurement IDs for measurements with "current" DataPoint (MetricData).
*/
private HashSet<Integer> getMidsWithinAllowedDataWindow(final List<DataPoint> states, final long now) {
HashSet<Integer> mids = new HashSet<Integer>();
//int i = 0;
for (Iterator<DataPoint> it = states.iterator(); it.hasNext(); ) {
DataPoint pt = it.next();
long timestamp = pt.getTimestamp();
// only allow data for the last MAX_DATA_BACKLOG_TIME ms
// this way we don't have to bring too much into memory which could
// severely impact performance
if ((now - timestamp) > MAX_DATA_BACKLOG_TIME) {
it.remove();
long days = (now - timestamp) / MeasurementConstants.DAY;
log.warn(" Avail measurement came in " + days + " days " + " late, dropping: timestamp=" + timestamp +
" measId=" + pt.getMeasurementId() + " value=" + pt.getMetricValue());
continue;
}
Integer mId = pt.getMeasurementId();
if (!mids.contains(mId)) {
mids.add(mId);
}
}
return mids;
}
private void updateDup(DataPoint state, AvailabilityDataRLE dup,
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails, Map<DataPoint, AvailabilityDataRLE> createMap,
Map<DataPoint, AvailabilityDataRLE> removeMap) throws BadAvailStateException {
if (dup.getAvailVal() == state.getValue()) {
// nothing to do
} else if (dup.getAvailVal() != AVAIL_DOWN) {
String msg = "New DataPoint and current DB value for " + "MeasurementId " + state.getMeasurementId() +
" / timestamp " + state.getTimestamp() + " have conflicting states. " +
"Since a non-zero rle value cannot be overridden, no update." + "\ncurrent rle value -> " +
dup +
// ask Juilet Sierra why (js) is here
":(js):\npoint trying to override current rle -> " + state;
throw new BadAvailStateException(msg);
} else {
Measurement meas = dup.getMeasurement();
long newStartime = dup.getStartime() + meas.getInterval();
insertPointOnBoundry(dup, newStartime, state, currAvails, createMap, removeMap);
}
}
/**
* sets avail's startime to newStartime and prepends a new avail obj from
* avail.getStartime() to newStartime with a value of state.getValue() Used
* specifically for a point which collides with a RLE on its startime
*/
private void insertPointOnBoundry(AvailabilityDataRLE avail, long newStartime, DataPoint pt,
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails, Map<DataPoint, AvailabilityDataRLE> createMap,
Map<DataPoint, AvailabilityDataRLE> removeMap) throws BadAvailStateException {
if (newStartime <= avail.getStartime()) {
return;
}
Measurement meas = avail.getMeasurement();
if (avail.getEndtime() == MAX_AVAIL_TIMESTAMP) {
DataPoint tmp = availabilityCache.get(pt.getMeasurementId());
if (tmp == null || pt.getTimestamp() >= tmp.getTimestamp()) {
updateAvailVal(avail, pt.getValue(), currAvails, createMap, removeMap);
} else {
prependState(pt, avail, currAvails, createMap, removeMap);
}
} else if (newStartime < avail.getEndtime()) {
prependState(pt, avail, currAvails, createMap, removeMap);
} else if (newStartime > avail.getEndtime()) {
removeAvail(avail, currAvails, createMap, removeMap);
} else if (newStartime == avail.getEndtime()) {
AvailabilityDataRLE after = findAvailAfter(pt, currAvails);
if (after == null) {
throw new BadAvailStateException("Availability measurement_id=" + pt.getMeasurementId() +
" does not have a availability point after timestamp " +
pt.getTimestamp());
}
if (after.getAvailVal() == pt.getValue()) {
// resolve by removing the before obj, if it exists,
// and sliding back the start time of after obj
AvailabilityDataRLE before = findAvailBefore(pt, currAvails);
if (before == null) {
after = updateStartime(after, avail.getStartime(), currAvails, createMap, removeMap);
} else if (before.getAvailVal() == after.getAvailVal()) {
removeAvail(avail, currAvails, createMap, removeMap);
removeAvail(before, currAvails, createMap, removeMap);
after = updateStartime(after, before.getStartime(), currAvails, createMap, removeMap);
}
} else {
// newStartime == avail.getEndtime() &&
// newStartime == after.getStartime() &&
// newStartime < after.getEndtime() &&
// pt.getValue() != after.getAvailVal()
// therefore, need to push back startTime and set the value
long interval = meas.getInterval();
if ((after.getStartime() + interval) < after.getEndtime()) {
prependState(pt, after, currAvails, createMap, removeMap);
} else {
DataPoint afterPt = new DataPoint(meas.getId().intValue(), after.getAvailVal(), after.getStartime());
AvailabilityDataRLE afterAfter = findAvailAfter(afterPt, currAvails);
if (afterAfter.getAvailVal() == pt.getValue()) {
removeAvail(after, currAvails, createMap, removeMap);
afterAfter = updateStartime(afterAfter, pt.getTimestamp(), currAvails, createMap, removeMap);
} else {
updateAvailVal(after, pt.getValue(), currAvails, createMap, removeMap);
}
}
}
}
}
private AvailabilityDataRLE findAvail(DataPoint state, Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails) {
Integer mId = state.getMeasurementId();
Collection<AvailabilityDataRLE> rles = currAvails.get(mId);
long start = state.getTimestamp();
for (AvailabilityDataRLE rle : rles) {
if (rle.getStartime() == start) {
return rle;
}
}
return null;
}
private AvailabilityDataRLE findAvailAfter(DataPoint state, Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails) {
final Integer mId = state.getMeasurementId();
final TreeSet<AvailabilityDataRLE> rles = currAvails.get(mId);
final long start = state.getTimestamp();
final AvailabilityDataRLE tmp = new AvailabilityDataRLE();
// tailSet is inclusive so we need to add 1 to start
tmp.setStartime(start + 1);
final SortedSet<AvailabilityDataRLE> set = rles.tailSet(tmp);
if (set.size() == 0) {
return null;
}
return (AvailabilityDataRLE) set.first();
}
private AvailabilityDataRLE findAvailBefore(DataPoint state, Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails) {
Integer mId = state.getMeasurementId();
TreeSet<AvailabilityDataRLE> rles = currAvails.get(mId);
long start = state.getTimestamp();
AvailabilityDataRLE tmp = new AvailabilityDataRLE();
// headSet is inclusive so we need to subtract 1 from start
tmp.setStartime(start - 1);
SortedSet<AvailabilityDataRLE> set = rles.headSet(tmp);
if (set.size() == 0) {
return null;
}
return set.last();
}
private void merge(DataPoint state, Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails,
Map<DataPoint, AvailabilityDataRLE> createMap,
Map<DataPoint, AvailabilityDataRLE> removeMap)
throws BadAvailStateException {
AvailabilityDataRLE dup = findAvail(state, currAvails);
if (dup != null) {
updateDup(state, dup, currAvails, createMap, removeMap);
return;
}
AvailabilityDataRLE before = findAvailBefore(state, currAvails);
AvailabilityDataRLE after = findAvailAfter(state, currAvails);
if (before == null && after == null) {
// this shouldn't happen here unless the resource was deleted
Measurement meas = getMeasurement(state.getMeasurementId());
// HHQ-5648 measurement could have been deleted
if (meas == null) {
return;
}
create(meas, state.getTimestamp(), state.getValue(), currAvails, createMap);
} else if (before == null) {
if (after.getAvailVal() != state.getValue()) {
prependState(state, after, currAvails, createMap, removeMap);
} else {
after = updateStartime(after, state.getTimestamp(), currAvails, createMap, removeMap);
}
} else if (after == null) {
// this shouldn't happen here
updateState(state, currAvails, createMap, removeMap);
} else {
insertAvail(before, after, state, currAvails, createMap, removeMap);
}
}
private void insertAvail(AvailabilityDataRLE before, AvailabilityDataRLE after, DataPoint state,
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails, Map<DataPoint, AvailabilityDataRLE> createMap,
Map<DataPoint, AvailabilityDataRLE> removeMap) {
if (state.getValue() != after.getAvailVal() && state.getValue() != before.getAvailVal()) {
Measurement meas = getMeasurement(state.getMeasurementId());
long pivotTime = state.getTimestamp() + meas.getInterval();
create(meas, state.getTimestamp(), pivotTime, state.getValue(), currAvails, createMap);
updateEndtime(before, state.getTimestamp());
after = updateStartime(after, pivotTime, currAvails, createMap, removeMap);
} else if (state.getValue() == after.getAvailVal() && state.getValue() != before.getAvailVal()) {
updateEndtime(before, state.getTimestamp());
after = updateStartime(after, state.getTimestamp(), currAvails, createMap, removeMap);
} else if (state.getValue() != after.getAvailVal() && state.getValue() == before.getAvailVal()) {
// this is fine
} else if (state.getValue() == after.getAvailVal() && state.getValue() == before.getAvailVal()) {
// this should never happen or else there is something wrong
// in the code
String msg = "AvailabilityData [" + before + "] and [" + after
+ "] have the same values. This should not be the case. " + "Cleaning up";
log.warn(msg);
updateEndtime(before, after.getEndtime());
removeAvail(after, currAvails, createMap, removeMap);
}
}
private boolean prependState(DataPoint state, AvailabilityDataRLE avail,
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails, Map<DataPoint, AvailabilityDataRLE> createMap,
Map<DataPoint, AvailabilityDataRLE> removeMap) {
AvailabilityDataRLE before = findAvailBefore(state, currAvails);
Measurement meas = avail.getMeasurement();
if (before != null && before.getAvailVal() == state.getValue()) {
long newStart = state.getTimestamp() + meas.getInterval();
updateEndtime(before, newStart);
avail = updateStartime(avail, newStart, currAvails, createMap, removeMap);
} else {
long newStart = state.getTimestamp() + meas.getInterval();
long endtime = newStart;
avail = updateStartime(avail, newStart, currAvails, createMap, removeMap);
create(avail.getMeasurement(), state.getTimestamp(), endtime, state.getValue(), currAvails, createMap);
}
return true;
}
private void updateAvailVal(AvailabilityDataRLE avail, double val,
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails, Map<DataPoint, AvailabilityDataRLE> createMap,
Map<DataPoint, AvailabilityDataRLE> removeMap) {
Measurement meas = avail.getMeasurement();
DataPoint state = new DataPoint(meas.getId().intValue(), val, avail.getStartime());
AvailabilityDataRLE before = findAvailBefore(state, currAvails);
if (before == null || before.getAvailVal() != val) {
avail.setAvailVal(val);
} else {
removeAvail(before, currAvails, createMap, removeMap);
avail = updateStartime(avail, before.getStartime(), currAvails, createMap, removeMap);
avail.setAvailVal(val);
}
}
private void updateEndtime(AvailabilityDataRLE avail, long endtime) {
avail.setEndtime(endtime);
}
private AvailabilityDataRLE updateStartime(AvailabilityDataRLE avail, long start,
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails, Map<DataPoint, AvailabilityDataRLE> createMap,
Map<DataPoint, AvailabilityDataRLE> removeMap) {
// this should not be the case here, but want to make sure and
// avoid HibernateUniqueKeyExceptions :(
AvailabilityDataRLE tmp;
Measurement meas = avail.getMeasurement();
Integer mId = meas.getId();
DataPoint tmpState = new DataPoint(mId.intValue(), avail.getAvailVal(), start);
if (null != (tmp = findAvail(tmpState, currAvails))) {
removeAvail(tmp, currAvails, createMap, removeMap);
}
removeAvail(avail, currAvails, createMap, removeMap);
return create(meas, start, avail.getEndtime(), avail.getAvailVal(), currAvails, createMap);
}
private void removeAvail(AvailabilityDataRLE avail, Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails,
Map<DataPoint, AvailabilityDataRLE> createMap, Map<DataPoint, AvailabilityDataRLE> removeMap) {
long start = avail.getStartime();
Integer mId = avail.getMeasurement().getId();
TreeSet<AvailabilityDataRLE> rles = currAvails.get(mId);
if (rles.remove(avail)) {
DataPoint key = new DataPoint(mId.intValue(), avail.getAvailVal(), start);
createMap.remove(key);
removeMap.put(key, avail);
}
}
private AvailabilityDataRLE getLastAvail(DataPoint state, Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails) {
Integer mId = state.getMeasurementId();
TreeSet<AvailabilityDataRLE> rles = currAvails.get(mId);
if (rles.size() == 0) {
return null;
}
return rles.last();
}
private AvailabilityDataRLE create(Measurement meas, long start, long end, double val,
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails, Map<DataPoint, AvailabilityDataRLE> createMap) {
AvailabilityDataRLE rtn = _createAvail(meas, start, end, val);
createMap.put(new DataPoint(meas.getId().intValue(), val, start), rtn);
Integer mId = meas.getId();
Collection<AvailabilityDataRLE> rles = currAvails.get(mId);
rles.add(rtn);
return rtn;
}
private AvailabilityDataRLE _createAvail(Measurement meas, long start, long end, double val) {
AvailabilityDataRLE rtn = new AvailabilityDataRLE();
rtn.setMeasurement(meas);
rtn.setStartime(start);
rtn.setEndtime(end);
rtn.setAvailVal(val);
return rtn;
}
private AvailabilityDataRLE create(Measurement meas, long start, double val,
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails, Map<DataPoint, AvailabilityDataRLE> createMap) {
AvailabilityDataRLE rtn = _createAvail(meas, start, MAX_AVAIL_TIMESTAMP, val);
createMap.put(new DataPoint(meas.getId().intValue(), val, start), rtn);
Integer mId = meas.getId();
Collection<AvailabilityDataRLE> rles = currAvails.get(mId);
// I am assuming that this will be cleaned up by the caller where it
// will update the rle before rtn if one exists
rles.add(rtn);
return rtn;
}
private boolean updateState(DataPoint state, Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails,
Map<DataPoint, AvailabilityDataRLE> createMap, Map<DataPoint, AvailabilityDataRLE> removeMap)
throws BadAvailStateException {
AvailabilityDataRLE avail = getLastAvail(state, currAvails);
final boolean debug = log.isDebugEnabled();
long begin = -1;
if (avail == null) {
Measurement meas = getMeasurement(state.getMeasurementId());
create(meas, state.getTimestamp(), state.getValue(), currAvails, createMap);
return true;
} else if (state.getTimestamp() < avail.getStartime()) {
if (debug) {
begin = System.currentTimeMillis();
}
merge(state, currAvails, createMap, removeMap);
if (debug) {
long now = System.currentTimeMillis();
log.debug("updateState.merge() -> " + (now - begin) + " ms");
}
return false;
} else if (state.getTimestamp() == avail.getStartime() && state.getValue() != avail.getAvailVal()) {
if (debug) {
begin = System.currentTimeMillis();
}
updateDup(state, avail, currAvails, createMap, removeMap);
if (debug) {
long now = System.currentTimeMillis();
log.debug("updateState.updateDup() -> " + (now - begin) + " ms");
}
return true;
} else if (state.getValue() == avail.getAvailVal()) {
if (debug) {
log.debug("no update state == avail " + state + " == " + avail);
}
return true;
}
if (debug) {
log.debug("updating endtime on avail -> " + avail + ", updating to state -> " + state);
}
updateEndtime(avail, state.getTimestamp());
create(avail.getMeasurement(), state.getTimestamp(), state.getValue(), currAvails, createMap);
return true;
}
/**
* update DB and availabilityCache with the changes marked in list states.
* Create/Remove is not actually done here - should call flushCreateAndRemove for the createMap and removeMap.
* @param states - States to update.
* @param currAvails - current DB state for measurement IDs.
* @param createMap - in/out param. filled with new AvailabilityDataRLEs
* @param removeMap - in/out param. filled with AvailabilityDataRLEs to remove.
*/
private void updateStates(List<DataPoint> states, Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails,
Map<DataPoint, AvailabilityDataRLE> createMap, Map<DataPoint, AvailabilityDataRLE> removeMap) {
if (states.size() == 0) {
return;
}
// as a performance optimization, fetch all the last avails
// at once, rather than one at a time in updateState()
final StopWatch watch = new StopWatch();
final boolean debug = log.isDebugEnabled();
int numUpdates = 0;
for (DataPoint state : states) {
try {
// need to check again since there could be multiple
// states with the same id in the list
DataPoint currState = availabilityCache.get(state.getMeasurementId());
if (currState != null && currState.getValue() == state.getValue()) {
continue;
}
boolean updateCache = updateState(state, currAvails, createMap, removeMap);
if (debug) {
log.debug("state " + state + " was updated, cache updated: " + updateCache);
}
if (updateCache) {
availabilityCache.put(state.getMeasurementId(), state);
numUpdates++;
}
} catch (BadAvailStateException e) {
log.warn(e.getMessage());
}
}
if (debug) {
log.debug("AvailabilityInserter updateStates: " + watch + ", points {total=" + states.size()
+ ", updateCache=" + numUpdates + "}");
}
}
/**
* update DB with the changes marked in list states.
* do not update the cache here, the timestamp is out of order.
* Create/Remove is not actually done here - should call flushCreateAndRemove for the createMap and removeMap.
* @param outOfOrderAvail - States that are not synched with the cache.
* @param currAvails - current DB state for measurement IDs.
* @param createMap - in/out param. filled with new AvailabilityDataRLEs
* @param removeMap - in/out param. filled with AvailabilityDataRLEs to remove.
*/
private void updateOutOfOrderState(List<DataPoint> outOfOrderAvail,
Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails, Map<DataPoint, AvailabilityDataRLE> createMap,
Map<DataPoint, AvailabilityDataRLE> removeMap) {
if (outOfOrderAvail.size() == 0) {
return;
}
final StopWatch watch = new StopWatch();
int numBadAvailState = 0;
for (DataPoint state : outOfOrderAvail) {
try {
// do not update the cache here, the timestamp is out of order
merge(state, currAvails, createMap, removeMap);
} catch (BadAvailStateException e) {
numBadAvailState++;
log.warn(e.getMessage());
}
}
if (log.isDebugEnabled()) {
log.debug("AvailabilityInserter updateOutOfOrderState: " + watch + ", points {total="
+ outOfOrderAvail.size() + ", badAvailState=" + numBadAvailState + "}");
}
}
/**
* update the cache with states that are equal to current state and have a newer timestamp.
* return 2 lists for updating - updateList and outOfOrderAvail.
* @param availPoints - DataPoint list that represents availability status of resources.
* @return updateList - Old and new states have different values and cache needs to be updated.
* @return outOfOrderAvail - Old metric has a newer timestamp
*/
private void updateCache(List<DataPoint> availPoints, List<DataPoint> updateList, List<DataPoint> outOfOrderAvail) {
if (availPoints.size() == 0) {
return;
}
final StopWatch watch = new StopWatch();
final boolean debug = log.isDebugEnabled();
for (DataPoint pt : availPoints) {
Integer meas_id = pt.getMeasurementId();
MetricValue mval = pt.getMetricValue();
double val = mval.getValue();
long timestamp = mval.getTimestamp();
DataPoint newState = new DataPoint(pt);
DataPoint oldState = availabilityCache.get(meas_id);
// we do not want to update the state if it changes
// instead change it when the db is changed in order
// to ensure the state of memory to db
// ONLY update memory state here if there is no change
AuthzSubject overlord = authzSubjectManager.getOverlordPojo();
AppdefEntityID entityId = null;
MaintenanceEvent mev=null;
if(oldState != null){
entityId = getMeasurement(oldState.getMeasurementId()).getEntityId();
try {
mev = PermissionManagerFactory.getInstance().getMaintenanceEventManager().getMaintenanceEvent(overlord, entityId);
} catch (Exception e) {
log.error("Cannot determine maintenance window for resource " + entityId.getId()+ ". Exception: " + e.getMessage());
}
}
// check if the "new" state is actually older than the state saved in the cache
// if so - OUT_OF_ORDER
if (oldState != null && timestamp < oldState.getTimestamp()) {
outOfOrderAvail.add(newState);
}
// else - check if the new old state is null or has a different value than the cache.
else if (oldState == null || oldState.getValue() == AVAIL_NULL || oldState.getValue() != val) {
//if there is no scheduled maintenance event
if(mev==null){
//just update with the new value
updateList.add(newState);
if (debug) {
String msg = "value of state[" + newState + "] differs from" + " current value["
+ ((oldState != null) ? oldState.toString() : "old state does not exist") + "]";
log.debug(msg);
}
}
else { // Otherwise, the data point belongs to a resource that is in scheduled maintenance window
if (mev.getStartTime() > timestamp) {
//maintenance window hasn't started yet, update state if necessary
if((oldState == null) || (oldState.getValue()!=val)) {
updateList.add(newState);
}
}
// if we're still in the maintenance window, make sure we're still in the defined time
else if (mev.getStartTime() < timestamp && timestamp < mev.getEndTime()) {
if((oldState == null) || (oldState.getValue()!=AVAIL_PAUSED)) { //no need to update if already in planned maintenance mode
updateList.add(new DataPoint(meas_id, AVAIL_PAUSED, mev.getStartTime()));
log.info("Marking resource " + entityId.getId() + " as being in scheduled maintenance mode.");
}
}
else if (timestamp > mev.getEndTime()) {
updateList.add(new DataPoint(meas_id, val, mev.getEndTime()));
log.info("Detected end of scheduled maintenance window for resource " + entityId.getId() +
". Resetting state to " + (val==AVAIL_UP ? "Available" : "Unavailable (" + val + ")"));
}
}
}
else {
availabilityCache.put(meas_id, newState);
}
}
if (debug) {
log.debug("AvailabilityInserter updateCache: " + watch + ", points {total=" + availPoints.size()
+ ", outOfOrder=" + outOfOrderAvail.size() + ", updateToDb=" + updateList.size()
+ ", updateCacheTimestamp=" + (availPoints.size() - outOfOrderAvail.size() - updateList.size())
+ "}");
}
}
private void checkAvailabilityState(List<DataPoint> availPoints) {
// The following code (until the creation of the StopWatch) is a fix for Jira issue [HHQ-5524].
// There is a strange scenario where the cache holds an availability metric with 'available' value while
// in the database the same availability metric value is 'not available' (in the HQ_AVAIL_DATA_RLE table).
// In that case we need to clear the availability cache because it is out of sync with the database.
// It is very hard to understand what causes this issue but it happens a lot in scale environments.
// If we don't clear the cache the resource which this availability
// metric belongs to will appear to be 'down' while it is not,
// clearing the cache causes the metric to be updated in the database to the correct value.
final List<Integer> includes = new ArrayList<Integer>(availPoints.size());
for (final DataPoint point : availPoints) {
includes.add(point.getMeasurementId());
}
// Find all of the availPoints in the database and check if the same datapoint
// is not in sync with the cache
final List<AvailabilityDataRLE> avails = availabilityDataDAO.findLastAvail(includes);
final List<Integer> mids = new ArrayList<Integer>();
for (final AvailabilityDataRLE data : avails) {
final Integer mid = data.getMeasurement().getId();
final DataPoint stateInCache = availabilityCache.get(mid);
if (null != stateInCache && (stateInCache.getValue() != data.getAvailVal())) {
availabilityCache.remove(mid);
mids.add(mid);
}
}
if (!mids.isEmpty()) {
log.info("The state of the Availability cache is out of sync with the database for measurementIds='" + mids +
"', clearing these metrics from the Availability cache");
}
}
private void sendDataToEventHandlers(List<DataPoint> data) {
final StopWatch watch = new StopWatch();
final boolean debug = log.isDebugEnabled();
int maxCapacity = data.size();
ArrayList<MeasurementEvent> events = new ArrayList<MeasurementEvent>(maxCapacity);
Map<Integer, MeasurementEvent> downEvents = new HashMap<Integer, MeasurementEvent>(maxCapacity);
List<MeasurementZevent> zevents = new ArrayList<MeasurementZevent>(maxCapacity);
boolean allEventsInteresting = Boolean.getBoolean(ALL_EVENTS_INTERESTING_PROP);
if (debug)
watch.markTimeBegin("isTriggerInterested");
for (DataPoint dp : data) {
Integer metricId = dp.getMeasurementId();
MetricValue val = dp.getMetricValue();
MeasurementEvent event = new MeasurementEvent(metricId, val);
if (registeredTriggers.isTriggerInterested(event) || allEventsInteresting) {
measurementManager.buildMeasurementEvent(event);
if (event.getValue().getValue() == AVAIL_DOWN) {
AppdefEntityID eventResourceAEID = event.getResource();
Resource r = (null == eventResourceAEID ? null : resourceManager.findResource(eventResourceAEID));
if (r != null && !r.isInAsyncDeleteState()) {
downEvents.put(r.getId(), event);
}
} else {
events.add(event);
}
}
zevents.add(new MeasurementZevent(metricId.intValue(), val));
}
if (debug)
watch.markTimeEnd("isTriggerInterested");
if (!downEvents.isEmpty()) {
if (debug)
watch.markTimeBegin("suppressMeasurementEvents");
// Determine whether the measurement events can
// be suppressed as part of hierarchical alerting
PermissionManagerFactory.getInstance().getHierarchicalAlertingManager()
.suppressMeasurementEvents(downEvents, true);
if (debug)
watch.markTimeEnd("suppressMeasurementEvents");
if (!downEvents.isEmpty()) {
events.addAll(downEvents.values());
}
}
if (!events.isEmpty()) {
messenger.publishMessage(EventConstants.EVENTS_TOPIC, events);
}
if (!zevents.isEmpty()) {
ZeventManager.getInstance().enqueueEventsAfterCommit(zevents);
}
if (debug) {
log.debug("AvailabilityInserter sendDataToEventHandlers: " + watch + ", points {total=" + maxCapacity
+ ", downEvents=" + downEvents.size() + ", eventsToPublish=" + events.size()
+ ", zeventsToEnqueue=" + zevents.size() + "}");
}
}
/**
* This method should only be called by the AvailabilityCheckService and is
* used to filter availability data points based on hierarchical alerting
* rules.
*
*
*/
public void sendDataToEventHandlers(Map<Integer, DataPoint> data) {
int maxCapacity = data.size();
Map<Integer, MeasurementEvent> events = new HashMap<Integer, MeasurementEvent>(maxCapacity);
List<MeasurementZevent> zevents = new ArrayList<MeasurementZevent>(maxCapacity);
boolean allEventsInteresting = Boolean.getBoolean(ALL_EVENTS_INTERESTING_PROP);
for (Integer resourceIdKey : data.keySet()) {
DataPoint dp = data.get(resourceIdKey);
Integer metricId = dp.getMeasurementId();
MetricValue val = dp.getMetricValue();
MeasurementEvent event = new MeasurementEvent(metricId, val);
if (registeredTriggers.isTriggerInterested(event) || allEventsInteresting) {
measurementManager.buildMeasurementEvent(event);
events.put(resourceIdKey, event);
}
zevents.add(new MeasurementZevent(metricId.intValue(), val));
}
if (!events.isEmpty()) {
// Determine whether the measurement events can
// be suppressed as part of hierarchical alerting
PermissionManagerFactory.getInstance().getHierarchicalAlertingManager()
.suppressMeasurementEvents(events, false);
messagePublisher.publishMessage(EventConstants.EVENTS_TOPIC,
new ArrayList<MeasurementEvent>(events.values()));
}
if (!zevents.isEmpty()) {
ZeventManager.getInstance().enqueueEventsAfterCommit(zevents);
}
}
private Measurement getMeasurement(Integer mId) {
return measurementManager.getMeasurement(mId);
}
private Map<Integer, StringBuilder> captureCurrAvailState(Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails) {
if (!traceLog.isDebugEnabled()) {
return null;
}
Map<Integer, StringBuilder> rtn = new HashMap<Integer, StringBuilder>();
for (Map.Entry<Integer, TreeSet<AvailabilityDataRLE>> entry : currAvails.entrySet()) {
Integer mid = entry.getKey();
Collection<AvailabilityDataRLE> rles = entry.getValue();
StringBuilder buf = new StringBuilder("\n");
for (AvailabilityDataRLE rle : rles) {
buf.append(mid).append(" | ").append(rle.getStartime()).append(" | ").append(rle.getEndtime())
.append(" | ").append(rle.getAvailVal()).append("\n");
}
rtn.put(mid, buf);
}
return rtn;
}
private void logAvailState(Map<Integer, StringBuilder> availState, Integer mid) {
StringBuilder buf = (StringBuilder) availState.get(mid);
traceLog.debug(buf.toString());
}
private void logStates(List<DataPoint> states, Integer mid) {
StringBuilder log = new StringBuilder("\n");
for (DataPoint pt : states) {
if (!pt.getMeasurementId().equals(mid)) {
continue;
}
log.append(pt.getMeasurementId()).append(" | ").append(pt.getTimestamp()).append(" | ").append(
pt.getMetricValue()).append("\n");
}
traceLog.debug(log.toString());
}
private Integer isAvailDataRLEValid(Map<Integer, TreeSet<AvailabilityDataRLE>> currAvails) {
synchronized (availabilityCache) {
for (Map.Entry<Integer, TreeSet<AvailabilityDataRLE>> entry : currAvails.entrySet()) {
Integer mId = entry.getKey();
Collection<AvailabilityDataRLE> rles = entry.getValue();
if (!isAvailDataRLEValid(mId, availabilityCache.get(mId), rles)) {
return mId;
}
}
}
return null;
}
private boolean isAvailDataRLEValid(Integer measId, DataPoint lastPt, Collection<AvailabilityDataRLE> avails) {
AvailabilityDataRLE last = null;
Set<Long> endtimes = new HashSet<Long>();
for (AvailabilityDataRLE avail : avails) {
Long endtime = new Long(avail.getEndtime());
if (endtimes.contains(endtime)) {
log.error("list for MID=" + measId + " contains two or more of the same endtime=" + endtime);
return false;
}
endtimes.add(endtime);
if (last == null) {
last = avail;
continue;
}
if (last.getAvailVal() == avail.getAvailVal()) {
log.error("consecutive availpoints have the same value, " + "first={" + last + "}, last={" + avail
+ "}");
return false;
} else if (last.getEndtime() != avail.getStartime()) {
log.error("there are gaps in the availability table" + "first={" + last + "}, last={" + avail + "}");
return false;
} else if (last.getStartime() > avail.getStartime()) {
log.error("startime availability is out of order" + "first={" + last + "}, last={" + avail + "}");
return false;
} else if (last.getEndtime() > avail.getEndtime()) {
log.error("endtime availability is out of order" + "first={" + last + "}, last={" + avail + "}");
return false;
}
last = avail;
}
if (((DataPoint) availabilityCache.get(measId)).getValue() != lastPt.getValue()) {
log.error("last avail data point does not match cache");
return false;
}
return true;
}
@Transactional(readOnly = true)
public boolean platformIsAvailableOrUnknown(int agentId) {
final Agent agent = agentDAO.get(agentId);
if (agent == null) {
return false;
}
Resource resource = null;
Collection<Platform> platforms = agent.getPlatforms();
for (final Platform p : platforms) {
if (PlatformDetector.isSupportedPlatform(p.getResource().getPrototype().getName())) {
resource = p.getResource();
if (resource == null || resource.isInAsyncDeleteState()) {
return false;
}
break;
}
}
final Measurement m = measurementManager.getAvailabilityMeasurement(resource);
if (m == null) {
return true;
}
final MetricValue last = getLastAvail(m);
if (last == null) {
return true;
}
if (last.getValue() == MeasurementConstants.AVAIL_UP ||
last.getValue() == MeasurementConstants.AVAIL_UNKNOWN) {
return true;
}
return false;
}
}