/*
* NOTE: This copyright does *not* cover user programs that use HQ
* program services by normal system calls through the application
* program interfaces provided as part of the Hyperic Plug-in Development
* Kit or the Hyperic Client Development Kit - this is merely considered
* normal use of the program, and does *not* fall under the heading of
* "derived work".
*
* Copyright (C) [2004-2010], Hyperic, Inc.
* This file is part of HQ.
*
* HQ is free software; you can redistribute it and/or modify
* it under the terms version 2 of the GNU General Public License as
* published by the Free Software Foundation. This program is distributed
* in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
* even the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*/
package org.hyperic.hq.measurement.server.session;
import java.math.BigDecimal;
import java.sql.BatchUpdateException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.PostConstruct;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hyperic.hibernate.dialect.HQDialect;
import org.hyperic.hq.common.ProductProperties;
import org.hyperic.hq.common.SystemException;
import org.hyperic.hq.common.TimeframeBoundriesException;
import org.hyperic.hq.common.shared.HQConstants;
import org.hyperic.hq.common.shared.ServerConfigManager;
import org.hyperic.hq.common.util.MessagePublisher;
import org.hyperic.hq.events.EventConstants;
import org.hyperic.hq.events.ext.RegisteredTriggers;
import org.hyperic.hq.measurement.MeasurementConstants;
import org.hyperic.hq.measurement.TimingVoodoo;
import org.hyperic.hq.measurement.data.MeasurementDataSourceException;
import org.hyperic.hq.measurement.ext.MeasurementEvent;
import org.hyperic.hq.measurement.shared.AvailabilityManager;
import org.hyperic.hq.measurement.shared.DataManager;
import org.hyperic.hq.measurement.shared.HighLowMetricValue;
import org.hyperic.hq.measurement.shared.MeasRange;
import org.hyperic.hq.measurement.shared.MeasRangeObj;
import org.hyperic.hq.measurement.shared.MeasTabManagerUtil;
import org.hyperic.hq.measurement.shared.MeasurementManager;
import org.hyperic.hq.measurement.shared.TopNManager;
import org.hyperic.hq.plugin.system.TopReport;
import org.hyperic.hq.product.MetricValue;
import org.hyperic.hq.stats.ConcurrentStatsCollector;
import org.hyperic.hq.zevents.ZeventEnqueuer;
import org.hyperic.util.TimeUtil;
import org.hyperic.util.jdbc.DBUtil;
import org.hyperic.util.pager.PageControl;
import org.hyperic.util.pager.PageList;
import org.hyperic.util.timer.StopWatch;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.orm.hibernate3.HibernateTransactionManager;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
/**
* The DataManagerImpl can be used to retrieve measurement data points
*
*/
@Service
@Transactional
public class DataManagerImpl implements DataManager {
private static final String ERR_START = "Begin and end times must be positive";
private static final String ERR_END = "Start time must be earlier than end time";
private static final String LOG_CTX = DataManagerImpl.class.getName();
private final Log log = LogFactory.getLog(LOG_CTX);
// The boolean system property that makes all events interesting. This
// property is provided as a testing hook so we can flood the event
// bus on demand.
public static final String ALL_EVENTS_INTERESTING_PROP = "org.hq.triggers.all.events.interesting";
private static final BigDecimal MAX_DB_NUMBER = new BigDecimal("10000000000000000000000");
private static final long MINUTE = 60 * 1000, HOUR = 60 * MINUTE;
protected final static long HOUR_IN_MILLI = TimeUnit.MILLISECONDS.convert(1L, TimeUnit.HOURS);
protected final static long SIX_HOURS_IN_MILLI = TimeUnit.MILLISECONDS.convert(6L, TimeUnit.HOURS);
protected final static long DAY_IN_MILLI = TimeUnit.MILLISECONDS.convert(1L, TimeUnit.DAYS);
// Table names
private static final String TAB_DATA_1H = MeasurementConstants.TAB_DATA_1H;
private static final String TAB_DATA_6H = MeasurementConstants.TAB_DATA_6H;
private static final String TAB_DATA_1D = MeasurementConstants.TAB_DATA_1D;
private static final String TAB_MEAS = MeasurementConstants.TAB_MEAS;
private static final String DATA_MANAGER_INSERT_TIME = ConcurrentStatsCollector.DATA_MANAGER_INSERT_TIME;
// Error strings
private static final String ERR_INTERVAL = "Interval cannot be larger than the time range";
private static final String PLSQL =
"BEGIN " +
"INSERT INTO :table (measurement_id, timestamp, value) " +
"VALUES(?, ?, ?); " +
"EXCEPTION WHEN DUP_VAL_ON_INDEX THEN " +
"UPDATE :table SET VALUE = ? " +
"WHERE timestamp = ? and measurement_id = ?; " + "END; ";
// Save some typing
private static final int IND_MIN = MeasurementConstants.IND_MIN;
private static final int IND_AVG = MeasurementConstants.IND_AVG;
private static final int IND_MAX = MeasurementConstants.IND_MAX;
private static final int IND_CFG_COUNT = MeasurementConstants.IND_CFG_COUNT;
private final DBUtil dbUtil;
// Pager class name
private boolean confDefaultsLoaded = false;
// Purge intervals, loaded once on first invocation.
private long purgeRaw, purge1h, purge6h, purge1d;
private static final long HOURS_PER_MEAS_TAB = MeasTabManagerUtil.NUMBER_OF_TABLES_PER_DAY;
private static final String DATA_MANAGER_RETRIES_TIME = ConcurrentStatsCollector.DATA_MANAGER_RETRIES_TIME;
private final MeasurementDAO measurementDAO;
private final MeasurementManager measurementManager;
private final ServerConfigManager serverConfigManager;
private final AvailabilityManager availabilityManager;
private final MetricDataCache metricDataCache;
private final ZeventEnqueuer zeventManager;
private final MessagePublisher messagePublisher;
private final RegisteredTriggers registeredTriggers;
private final ConcurrentStatsCollector concurrentStatsCollector;
private final int transactionTimeout;
private final TopNManager topNManager;
@Autowired
public DataManagerImpl(DBUtil dbUtil, MeasurementDAO measurementDAO,
MeasurementManager measurementManager,
ServerConfigManager serverConfigManager,
AvailabilityManager availabilityManager,
MetricDataCache metricDataCache, ZeventEnqueuer zeventManager,
MessagePublisher messagePublisher, RegisteredTriggers registeredTriggers,
ConcurrentStatsCollector concurrentStatsCollector,
HibernateTransactionManager transactionManager,
TopNManager topNManager) {
this.dbUtil = dbUtil;
this.measurementDAO = measurementDAO;
this.measurementManager = measurementManager;
this.serverConfigManager = serverConfigManager;
this.availabilityManager = availabilityManager;
this.metricDataCache = metricDataCache;
this.zeventManager = zeventManager;
this.messagePublisher = messagePublisher;
this.registeredTriggers = registeredTriggers;
this.concurrentStatsCollector = concurrentStatsCollector;
this.transactionTimeout = transactionManager.getDefaultTimeout();
this.topNManager = topNManager;
}
@PostConstruct
public void initStatsCollector() {
concurrentStatsCollector.register(DATA_MANAGER_INSERT_TIME);
concurrentStatsCollector.register(DATA_MANAGER_RETRIES_TIME);
}
private double getValue(ResultSet rs) throws SQLException {
double val = rs.getDouble("value");
if (rs.wasNull()) {
val = Double.NaN;
}
return val;
}
private void checkTimeArguments(long begin, long end) throws IllegalArgumentException {
if (begin > end) {
throw new IllegalArgumentException(ERR_END);
}
if (begin < 0) {
throw new IllegalArgumentException(ERR_START);
}
}
private HighLowMetricValue getMetricValue(ResultSet rs) throws SQLException {
long timestamp = rs.getLong("timestamp");
double value = this.getValue(rs);
if (!Double.isNaN(value)) {
try {
double high = rs.getDouble("peak");
double low = rs.getDouble("low");
return new HighLowMetricValue(value, high, low, timestamp);
} catch (SQLException e) {
// Peak and low columns do not exist
}
}
return new HighLowMetricValue(value, timestamp);
}
// Returns the next index to be used
private int setStatementArguments(PreparedStatement stmt, int start, Integer[] ids)
throws SQLException {
// Set ID's
int i = start;
for (Integer id : ids) {
stmt.setInt(i++, id.intValue());
}
return i;
}
private void checkTimeArguments(long begin, long end, long interval)
throws IllegalArgumentException {
checkTimeArguments(begin, end);
if (interval > (end - begin)) {
throw new IllegalArgumentException(ERR_INTERVAL);
}
}
/**
* Save the new MetricValue to the database
*
* @param mv the new MetricValue
* @throws NumberFormatException if the value from the
* DataPoint.getMetricValue() cannot instantiate a BigDecimal
*
*
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void addData(Integer mid, MetricValue mv, boolean overwrite) {
Measurement meas = measurementManager.getMeasurement(mid);
List<DataPoint> pts = Collections.singletonList(new DataPoint(meas.getId(), mv));
addData(pts, overwrite);
}
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void addData(List<DataPoint> data, String aggTable, Connection conn) throws Exception {
try {
insertDataWithOneInsert(data, aggTable, conn);
conn.commit();
} catch (Exception e) {
conn.rollback();
throw e;
} finally {
DBUtil.closeConnection(LOG_CTX, conn);
}
}
private void insertDataWithOneInsert(List<DataPoint> dpts, String table, Connection conn) {
Statement stmt = null;
ResultSet rs = null;
try {
StringBuilder values = new StringBuilder();
for (DataPoint pt : dpts) {
Integer metricId = pt.getMeasurementId();
HighLowMetricValue metricVal = (HighLowMetricValue) pt.getMetricValue();
BigDecimal val = new BigDecimal(metricVal.getValue());
BigDecimal highVal = new BigDecimal(metricVal.getHighValue());
BigDecimal lowVal = new BigDecimal(metricVal.getLowValue());
values.append("(").append(metricId.intValue()).append(", ").append(
metricVal.getTimestamp()).append(", ").append(
getDecimalInRange(val, metricId)).append(", ").append(
getDecimalInRange(lowVal, metricId)).append(", ").append(
getDecimalInRange(highVal, metricId)).append("),");
}
String sql = "insert into " + table + " (measurement_id, timestamp, value, minvalue, maxvalue)" +
" values " + values.substring(0, values.length() - 1);
stmt = conn.createStatement();
stmt.executeUpdate(sql);
} catch (SQLException e) {
// If there is a SQLException, then none of the data points
// should be inserted. Roll back the txn.
log.error("an unexpected SQL exception has occured inserting datapoints to the " + table + " table:\n",e);
} finally {
DBUtil.closeJDBCObjects(LOG_CTX, null, stmt, rs);
}
}
@Transactional(propagation = Propagation.REQUIRES_NEW)
public boolean addData(List<DataPoint> data) {
return this._addData(data,safeGetConnection());
}
public boolean addData(List<DataPoint> data, Connection conn) {
return this._addData(data,safeGetConnection());
}
@Transactional(propagation = Propagation.REQUIRES_NEW)
public boolean addTopData(List<TopNData> topNData) {
return this._addTopData(topNData, safeGetConnection());
}
/**
* Write metric data points to the DB with transaction
*
* @param data a list of {@link DataPoint}s
* @throws NumberFormatException if the value from the
* DataPoint.getMetricValue() cannot instantiate a BigDecimal
*
*
*/
protected boolean _addData(List<DataPoint> data, Connection conn) {
if (shouldAbortDataInsertion(data)) {
return true;
}
data = enforceUnmodifiable(data);
log.debug("Attempting to insert data in a single transaction.");
HQDialect dialect = measurementDAO.getHQDialect();
boolean succeeded = false;
final boolean debug = log.isDebugEnabled();
if (conn == null) {
return false;
}
try {
boolean autocommit = conn.getAutoCommit();
try {
final long start = System.currentTimeMillis();
conn.setAutoCommit(false);
if (dialect.supportsMultiInsertStmt()) {
succeeded = insertDataWithOneInsert(data, conn);
} else {
succeeded = insertDataInBatch(data, conn);
}
if (succeeded) {
conn.commit();
final long end = System.currentTimeMillis();
if (debug) {
log.debug("Inserting data in a single transaction " + "succeeded");
log.debug("Data Insertion process took " + (end - start) + " ms");
}
concurrentStatsCollector.addStat(end - start, DATA_MANAGER_INSERT_TIME);
sendMetricEvents(data);
} else {
if (debug) {
log.debug("Inserting data in a single transaction failed."
+ " Rolling back transaction.");
}
conn.rollback();
conn.setAutoCommit(true);
List<DataPoint> processed = addDataWithCommits(data, true, conn);
final long end = System.currentTimeMillis();
concurrentStatsCollector.addStat(end - start, DATA_MANAGER_INSERT_TIME);
sendMetricEvents(processed);
if (debug) {
log.debug("Data Insertion process took " + (end - start) + " ms");
}
}
} catch (SQLException e) {
conn.rollback();
throw e;
} finally {
conn.setAutoCommit(autocommit);
}
} catch (SQLException e) {
log.debug("Transaction failed around inserting metric data.", e);
} finally {
DBUtil.closeConnection(LOG_CTX, conn);
}
return succeeded;
}
protected boolean _addTopData(List<TopNData> topNData, Connection conn) {
if (log.isDebugEnabled()) {
log.debug("Attempting to insert topN data in a single transaction.");
}
boolean succeeded = false;
final boolean debug = log.isDebugEnabled();
if (conn == null) {
return false;
}
try {
boolean autocommit = conn.getAutoCommit();
try {
final long start = System.currentTimeMillis();
conn.setAutoCommit(false);
succeeded = insertTopData(conn, topNData, false);
if (succeeded) {
conn.commit();
final long end = System.currentTimeMillis();
if (debug) {
log.debug("Inserting TopN data in a single transaction " + "succeeded");
log.debug("TopN Data Insertion process took " + (end - start) + " ms");
}
concurrentStatsCollector.addStat(end - start, DATA_MANAGER_INSERT_TIME);
} else {
if (debug) {
log.debug("Inserting TopN data in a single transaction failed." + " Rolling back transaction" +
".");
}
conn.rollback();
}
} catch (SQLException e) {
conn.rollback();
throw e;
} finally {
conn.setAutoCommit(autocommit);
}
} catch (SQLException e) {
log.debug("Transaction failed around inserting TopN data.", e);
} finally {
DBUtil.closeConnection(LOG_CTX, conn);
}
return succeeded;
}
/**
* Write metric datapoints to the DB without transaction
*
* @param data a list of {@link DataPoint}s
* @param overwrite If true, attempt to over-write values when an insert of
* the data fails (i.e. it already exists). You may not want to
* over-write values when, for instance, the back filler is inserting
* data.
* @throws NumberFormatException if the value from the
* DataPoint.getMetricValue() cannot instantiate a BigDecimal
*
*
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void addData(List<DataPoint> data, boolean overwrite) {
/**
* We have to account for 2 types of metric data insertion here: 1 - New
* data, using 'insert' 2 - Old data, using 'update'
*
* We optimize the amount of DB roundtrips here by executing in batch,
* however there are some serious gotchas:
*
* 1 - If the 'insert' batch update fails, a BatchUpdateException can be
* thrown instead of just returning an error within the executeBatch()
* array. 2 - This is further complicated by the fact that some drivers
* will throw the exception at the first instance of an error, and some
* will continue with the rest of the batch.
*/
if (shouldAbortDataInsertion(data)) {
return;
}
log.debug("Attempting to insert/update data outside a transaction.");
data = enforceUnmodifiable(data);
Connection conn = safeGetConnection();
if (conn == null) {
log.debug("Inserting/Updating data outside a transaction failed.");
return;
}
try {
boolean autocommit = conn.getAutoCommit();
try {
conn.setAutoCommit(true);
addDataWithCommits(data, overwrite, conn);
} finally {
conn.setAutoCommit(autocommit);
}
} catch (SQLException e) {
log.debug("Inserting/Updating data outside a transaction failed "
+ "because autocommit management failed.", e);
} finally {
DBUtil.closeConnection(LOG_CTX, conn);
}
}
private List<DataPoint> addDataWithCommits(List<DataPoint> data, boolean overwrite, Connection conn) {
final StopWatch watch = new StopWatch();
try {
return _addDataWithCommits(data, overwrite, conn);
} finally {
concurrentStatsCollector.addStat(watch.getElapsed(), DATA_MANAGER_RETRIES_TIME);
}
}
private List<DataPoint> _addDataWithCommits(List<DataPoint> data, boolean overwrite, Connection conn) {
Set<DataPoint> failedToSaveMetrics = new HashSet<DataPoint>();
List<DataPoint> left = data;
while (!left.isEmpty()) {
int numLeft = left.size();
if (log.isDebugEnabled()) {
log.debug("Attempting to insert " + numLeft + " points");
}
try {
left = insertData(conn, left, true);
} catch (SQLException e) {
assert false : "The SQLException should not happen: " + e;
}
if (log.isDebugEnabled()) {
log.debug("Num left = " + left.size());
}
if (left.isEmpty()) {
break;
}
if (!overwrite) {
if (log.isDebugEnabled()) {
log.debug("We are not updating the remaining " + left.size() + " points");
}
failedToSaveMetrics.addAll(left);
break;
}
// The insert couldn't insert everything, so attempt to update
// the things that are left
if (log.isDebugEnabled()) {
log.debug("Sending " + left.size() + " data points to update");
}
left = updateData(conn, left);
if (left.isEmpty()) {
break;
}
if (log.isDebugEnabled()) {
log.debug("Update left " + left.size() + " points to process");
}
if (numLeft == left.size()) {
DataPoint remPt = left.remove(0);
failedToSaveMetrics.add(remPt);
// There are some entries that we weren't able to do
// anything about ... that sucks.
log.warn("Unable to do anything about " + numLeft + " data points. Sorry.");
log.warn("Throwing away data point " + remPt);
}
}
log.debug("Inserting/Updating data outside a transaction finished.");
return removeMetricsFromList(data, failedToSaveMetrics);
}
private boolean shouldAbortDataInsertion(List<?> data) {
if (data.isEmpty()) {
log.debug("Aborting data insertion since data list is empty. This is ok.");
return true;
} else {
return false;
}
}
private <T> List<T> enforceUnmodifiable(List<T> aList) {
return Collections.unmodifiableList(aList);
}
private List<DataPoint> removeMetricsFromList(List<DataPoint> data,
Set<DataPoint> metricsToRemove) {
if (metricsToRemove.isEmpty()) {
return data;
}
Set<DataPoint> allMetrics = new HashSet<DataPoint>(data);
allMetrics.removeAll(metricsToRemove);
return new ArrayList<DataPoint>(allMetrics);
}
/**
* Convert a decimal value to something suitable for being thrown into the
* database with a NUMERIC(24,5) definition
*/
private BigDecimal getDecimalInRange(BigDecimal val, Integer metricId) {
val = val.setScale(5, BigDecimal.ROUND_HALF_EVEN);
if (val.compareTo(MAX_DB_NUMBER) == 1) {
log.warn("Value [" + val + "] for metric id=" + metricId +
"is too big to put into the DB. Truncating to [" + MAX_DB_NUMBER + "]");
return MAX_DB_NUMBER;
}
return val;
}
private void sendMetricEvents(List<DataPoint> data) {
if (data.isEmpty()) {
return;
}
// Finally, for all the data which we put into the system, make sure
// we update our internal cache, kick off the events, etc.
final boolean debug = log.isDebugEnabled();
final StopWatch watch = new StopWatch();
if (debug) {
watch.markTimeBegin("analyzeMetricData");
}
analyzeMetricData(data);
if (debug) {
watch.markTimeEnd("analyzeMetricData");
}
Collection<DataPoint> cachedData = updateMetricDataCache(data);
sendDataToEventHandlers(cachedData);
if (debug) {
log.debug(watch);
}
}
private void analyzeMetricData(List<DataPoint> data) {
Analyzer analyzer = getAnalyzer();
if (analyzer != null) {
for (DataPoint dp : data) {
analyzer.analyzeMetricValue(dp.getMeasurementId(), dp.getMetricValue());
}
}
}
private Collection<DataPoint> updateMetricDataCache(List<DataPoint> data) {
return metricDataCache.bulkAdd(data);
}
private void sendDataToEventHandlers(Collection<DataPoint> data) {
ArrayList<MeasurementEvent> events = new ArrayList<MeasurementEvent>();
List<MeasurementZevent> zevents = new ArrayList<MeasurementZevent>();
boolean allEventsInteresting = Boolean.getBoolean(ALL_EVENTS_INTERESTING_PROP);
for (DataPoint dp : data) {
Integer metricId = dp.getMeasurementId();
MetricValue val = dp.getMetricValue();
MeasurementEvent event = new MeasurementEvent(metricId, val);
if (registeredTriggers.isTriggerInterested(event) || allEventsInteresting) {
measurementManager.buildMeasurementEvent(event);
events.add(event);
}
zevents.add(new MeasurementZevent(metricId.intValue(), val));
}
if (!events.isEmpty()) {
messagePublisher.publishMessage(EventConstants.EVENTS_TOPIC, events);
}
if (!zevents.isEmpty()) {
try {
// XXX: Shouldn't this be a transactional queueing?
zeventManager.enqueueEvents(zevents);
} catch (InterruptedException e) {
log.warn("Interrupted while sending events. Some data may " + "be lost");
}
}
}
private List<DataPoint> getRemainingDataPoints(List<DataPoint> data, int[] execInfo) {
List<DataPoint> res = new ArrayList<DataPoint>();
int idx = 0;
// this is the case for mysql
if (execInfo.length == 0) {
return res;
}
for (Iterator<DataPoint> i = data.iterator(); i.hasNext(); idx++) {
DataPoint pt = i.next();
if (execInfo[idx] == Statement.EXECUTE_FAILED) {
res.add(pt);
}
}
if (log.isDebugEnabled()) {
log.debug("Need to deal with " + res.size() + " unhandled " + "data points (out of " +
execInfo.length + ")");
}
return res;
}
private List<DataPoint> getRemainingDataPointsAfterBatchFail(List<DataPoint> data, int[] counts) {
List<DataPoint> res = new ArrayList<DataPoint>();
Iterator<DataPoint> i = data.iterator();
int idx;
for (idx = 0; idx < counts.length; idx++) {
DataPoint pt = i.next();
if (counts[idx] == Statement.EXECUTE_FAILED) {
res.add(pt);
}
}
if (log.isDebugEnabled()) {
log.debug("Need to deal with " + res.size() + " unhandled " + "data points (out of " +
counts.length + "). " + "datasize=" + data.size());
}
// It's also possible that counts[] is not as long as the list
// of data points, so we have to return all the un-processed points
if (data.size() != counts.length) {
res.addAll(data.subList(idx, data.size()));
}
return res;
}
/**
* Insert the metric data points to the DB with one insert statement. This
* should only be invoked when the DB supports multi-insert statements.
*
* @param data a list of {@link DataPoint}s
* @return <code>true</code> if the multi-insert succeeded;
* <code>false</code> otherwise.
*/
private boolean insertDataWithOneInsert(List<DataPoint> data, Connection conn) {
Statement stmt = null;
final Map<String, Set<DataPoint>> buckets = MeasRangeObj.getInstance().bucketDataEliminateDups(data);
final boolean debug = log.isDebugEnabled();
String sql = "";
final HQDialect dialect = measurementDAO.getHQDialect();
final boolean supportsAsyncCommit = dialect.supportsAsyncCommit();
try {
for (Entry<String, Set<DataPoint>> entry : buckets.entrySet()) {
final String table = entry.getKey();
final Set<DataPoint> dpts = entry.getValue();
final int dptsSize = dpts.size();
final StringBuilder values = new StringBuilder(dptsSize*15);
int rows = 0;
for (final DataPoint pt : dpts) {
final Integer metricId = pt.getMeasurementId();
final MetricValue val = pt.getMetricValue();
final BigDecimal bigDec = new BigDecimal(val.getValue());
rows++;
values.append("(").append(val.getTimestamp()).append(",")
.append(metricId.intValue()).append(",")
.append(getDecimalInRange(bigDec, metricId)).append(")");
if (rows < dptsSize) {
values.append(",");
}
}
sql = "insert into " + table + " (timestamp, measurement_id, value) values " + values;
stmt = conn.createStatement();
if (supportsAsyncCommit) {
stmt.execute(dialect.getSetAsyncCommitStmt(false));
}
final int rowsUpdated = stmt.executeUpdate(sql);
if (supportsAsyncCommit) {
stmt.execute(dialect.getSetAsyncCommitStmt(true));
}
stmt.close();
stmt = null;
if (debug) {
log.debug("Inserted " + rowsUpdated + " rows into " + table + " (attempted " + rows + " rows)");
}
if (rowsUpdated < rows) {
return false;
}
}
} catch (SQLException e) {
// If there is a SQLException, then none of the data points
// should be inserted. Roll back the txn.
if (debug) {
log.debug("Error inserting data with one insert stmt: " + e +
". Server will retry the insert in degraded mode. sql=\n" + sql);
}
return false;
} finally {
DBUtil.closeStatement(LOG_CTX, stmt);
}
return true;
}
/**
* Insert the metric data points to the DB in batch.
*
* @param data a list of {@link DataPoint}s
* @return <code>true</code> if the batch insert succeeded;
* <code>false</code> otherwise.
*/
private boolean insertDataInBatch(List<DataPoint> data, Connection conn) {
List<DataPoint> left = data;
try {
if (log.isDebugEnabled()) {
log.debug("Attempting to insert " + left.size() + " points");
}
left = insertData(conn, left, false);
if (log.isDebugEnabled()) {
log.debug("Num left = " + left.size());
}
if (!left.isEmpty()) {
if (log.isDebugEnabled()) {
log.debug("Need to update " + left.size() + " data points.");
log.debug("Data points to update: " + left);
}
return false;
}
} catch (SQLException e) {
// If there is a SQLException, then none of the data points
// should be inserted. Roll back the txn.
if (log.isDebugEnabled()) {
log.debug("Error while inserting data in batch (this is ok) " + e.getMessage());
}
return false;
}
return true;
}
/**
* Retrieve a DB connection.
*
* @return The connection or <code>null</code>.
*/
private Connection safeGetConnection() {
Connection conn = null;
try {
// XXX: may want to explore grabbing a connection directly from the transactionManager
conn = dbUtil.getConnection();
} catch (SQLException e) {
log.error("Failed to retrieve connection from data source", e);
}
return conn;
}
/**
* This method inserts data into the data table. If any data points in the
* list fail to get added (e.g. because of a constraint violation), it will
* be returned in the result list.
*
* @param conn The connection.
* @param data The data points to insert.
* @param continueOnSQLException <code>true</code> to continue inserting the
* rest of the data points even after a <code>SQLException</code>
* occurs; <code>false</code> to throw the <code>SQLException</code>.
* @return The list of data points that were not inserted.
* @throws SQLException only if there is an exception for one of the data
* point batch inserts and <code>continueOnSQLException</code> is
* set to <code>false</code>.
*/
private List<DataPoint> insertData(Connection conn, List<DataPoint> data,
boolean continueOnSQLException)
throws SQLException {
PreparedStatement stmt = null;
final List<DataPoint> left = new ArrayList<DataPoint>();
final Map<String, List<DataPoint>> buckets = MeasRangeObj.getInstance().bucketData(data);
final HQDialect dialect = measurementDAO.getHQDialect();
final boolean supportsDupInsStmt = dialect.supportsDuplicateInsertStmt();
final boolean supportsPLSQL = dialect.supportsPLSQL();
final StringBuilder buf = new StringBuilder();
for (final Entry<String, List<DataPoint>> entry : buckets.entrySet()) {
buf.setLength(0);
final String table = entry.getKey();
final List<DataPoint> dpts = entry.getValue();
try {
if (supportsDupInsStmt) {
stmt = conn.prepareStatement(
buf.append("INSERT INTO ").append(table)
.append(" (measurement_id, timestamp, value) VALUES (?, ?, ?)")
.append(" ON DUPLICATE KEY UPDATE value = ?")
.toString());
} else if (supportsPLSQL) {
final String sql = PLSQL.replaceAll(":table", table);
stmt = conn.prepareStatement(sql);
} else {
stmt = conn.prepareStatement(
buf.append("INSERT INTO ")
.append(table)
.append(" (measurement_id, timestamp, value) VALUES (?, ?, ?)")
.toString());
}
// TODO need to set synchronous commit to off
for (DataPoint pt : dpts) {
Integer metricId = pt.getMeasurementId();
MetricValue val = pt.getMetricValue();
BigDecimal bigDec;
bigDec = new BigDecimal(val.getValue());
stmt.setInt(1, metricId.intValue());
stmt.setLong(2, val.getTimestamp());
stmt.setBigDecimal(3, getDecimalInRange(bigDec, metricId));
if (supportsDupInsStmt) {
stmt.setBigDecimal(4, getDecimalInRange(bigDec, metricId));
} else if (supportsPLSQL) {
stmt.setBigDecimal(4, getDecimalInRange(bigDec, metricId));
stmt.setLong(5, val.getTimestamp());
stmt.setInt(6, metricId.intValue());
}
stmt.addBatch();
}
int[] execInfo = stmt.executeBatch();
left.addAll(getRemainingDataPoints(dpts, execInfo));
} catch (BatchUpdateException e) {
if (!continueOnSQLException) {
throw e;
}
left.addAll(getRemainingDataPointsAfterBatchFail(dpts, e.getUpdateCounts()));
} catch (SQLException e) {
if (!continueOnSQLException) {
throw e;
}
// If the batch insert is not within a transaction, then we
// don't know which of the inserts completed successfully.
// Assume they all failed.
left.addAll(dpts);
if (log.isDebugEnabled()) {
log.debug("A general SQLException occurred during the insert. " +
"Assuming that none of the " + dpts.size() +
" data points were inserted.", e);
}
} finally {
DBUtil.closeStatement(LOG_CTX, stmt);
}
}
return left;
}
public int purgeTopNData(Date timeToKeep) {
PreparedStatement ps = null;
int topNDeleted = -1;
Connection conn = safeGetConnection();
try {
ps = conn.prepareStatement("select drop_old_partitions(?, ?);");
ps.setString(1, TopNData.class.getSimpleName());
ps.setTimestamp(2, new Timestamp(timeToKeep.getTime()));
ResultSet rs = ps.executeQuery();
rs.next();
topNDeleted = rs.getInt(1);
} catch (SQLException e) {
log.error("Problem purging old TopN Data", e);
} finally {
DBUtil.closeStatement(LOG_CTX, ps);
}
return topNDeleted;
}
/*
* (non-Javadoc)
*
* @see org.hyperic.hq.measurement.shared.DataManager#getTopNData(int, long)
*/
public TopNData getTopNData(int resourceId, long time) {
Statement stmt = null;
Connection conn = safeGetConnection();
StringBuilder builder = new StringBuilder();
try {
builder.append("SELECT data FROM TOPNDATA TOP WHERE TOP.resourceid='").append(resourceId)
.append("' AND TOP.time = '").append(new java.sql.Timestamp(time)).append("'");
stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(builder.toString());
while (rs.next()) {
TopNData data = new TopNData();
data.setData(rs.getBytes("data"));
data.setResourceId(resourceId);
data.setTime(new Date(time));
return data;
}
} catch (SQLException e) {
log.error("Problem fetching TOP data", e);
} finally {
DBUtil.closeStatement(LOG_CTX, stmt);
}
return null;
}
/*
* (non-Javadoc)
*
* @see
* org.hyperic.hq.measurement.shared.DataManager#getTopNDataAsString(int,
* long)
*/
@Transactional(readOnly = true)
public TopReport getTopReport(int resourceId, long time) {
TopNData data = getTopNData(resourceId, time);
if(data == null){
return null;
}
try {
byte[] unCompressedData = topNManager.uncompressData(data.getData());
return TopReport.fromSerializedForm(unCompressedData);
} catch (Exception e) {
log.error("Error un serializing TopN data", e);
}
return null;
}
/*
* (non-Javadoc)
*
* @see
* org.hyperic.hq.measurement.shared.DataManager#getLatestAvailableTopDataTimes
* (int, int)
*/
@Transactional(readOnly = true)
public List<Long> getLatestAvailableTopDataTimes(int resourceId, int count) {
List<Long> times = new ArrayList<Long>();
Statement stmt = null;
Connection conn = safeGetConnection();
StringBuilder builder = new StringBuilder();
try {
builder.append("SELECT TIME FROM TOPNDATA TOP WHERE TOP.resourceid='").append(resourceId)
.append("' ORDER BY time DESC LIMIT ").append(count);
stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(builder.toString());
while (rs.next()) {
times.add(rs.getTimestamp("time").getTime());
}
} catch (SQLException e) {
log.error("Problem fetching TOP data", e);
return times;
} finally {
DBUtil.closeStatement(LOG_CTX, stmt);
}
return times;
}
/*
* (non-Javadoc)
*
* @see
* org.hyperic.hq.measurement.shared.DataManager#getAvailableTopDataTimes
* (int, long, long)
*/
@Transactional(readOnly=true)
public List<Long> getAvailableTopDataTimes(int resourceId, long from, long to) {
List<Long> times = new ArrayList<Long>();
Statement stmt = null;
Connection conn = safeGetConnection();
StringBuilder builder = new StringBuilder();
try {
builder.append("SELECT TIME FROM TOPNDATA TOP WHERE TOP.resourceid='").append(resourceId)
.append("' AND TOP.time BETWEEN '").append(new java.sql.Timestamp(from)).append("' AND '")
.append(new java.sql.Timestamp(to)).append("'");
stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(builder.toString());
while (rs.next()) {
times.add(rs.getTimestamp("time").getTime());
}
} catch (SQLException e) {
log.error("Problem fetching TOP data", e);
return times;
} finally {
DBUtil.closeStatement(LOG_CTX, stmt);
}
return times;
}
private boolean insertTopData(Connection conn, List<TopNData> topNData, boolean continueOnSQLException) throws
SQLException {
if (log.isDebugEnabled()) {
log.debug("Inserting Top Processes data - '" + topNData + "'");
}
PreparedStatement stmt = null;
final StringBuilder buf = new StringBuilder();
Map<Date, List<TopNData>> dayToDataMap = mapDayToData(topNData);
for (Entry<Date, List<TopNData>> entry : dayToDataMap.entrySet()) {
try {
String partitionName = getAndCreatePartition(TopNData.class.getSimpleName(), entry.getKey(), conn);
stmt = conn.prepareStatement(buf.append("INSERT INTO ").append(partitionName).
append(" (resourceId, time, data) VALUES (?, ?, ?)").toString());
for (TopNData data : entry.getValue()) {
stmt.setInt(1, data.getResourceId());
stmt.setTimestamp(2, new java.sql.Timestamp(data.getTime().getTime()));
stmt.setBytes(3, data.getData());
stmt.addBatch();
}
int[] execInfo = stmt.executeBatch();
} catch (BatchUpdateException e) {
if (!continueOnSQLException) {
throw e;
}
} catch (SQLException e) {
if (!continueOnSQLException) {
throw e;
}
if (log.isDebugEnabled()) {
log.debug("A general SQLException occurred during the insert of TopN. ", e);
}
} finally {
DBUtil.closeStatement(LOG_CTX, stmt);
}
}
return true;
}
private Map<Date, List<TopNData>> mapDayToData(List<TopNData> topNData) {
Map<Date, List<TopNData>> dayToDataMap = new HashMap<Date, List<TopNData>>();
for (TopNData data : topNData) {
Date day = DateUtils.truncate(data.getTime(), Calendar.DAY_OF_MONTH);
List<TopNData> datas = dayToDataMap.get(day);
if (datas == null) {
dayToDataMap.put(day, new LinkedList<TopNData>());
}
dayToDataMap.get(day).add(data);
}
return dayToDataMap;
}
private String getAndCreatePartition(final String tableName, final Date time,
final Connection conn) throws SQLException {
PreparedStatement ps = null;
try {
ps = conn.prepareStatement("select get_and_create_partition(?,?);");
ps.setString(1, tableName);
ps.setTimestamp(2, new Timestamp(time.getTime()));
final ResultSet rs = ps.executeQuery();
rs.next();
return rs.getString(1);
} finally {
DBUtil.closeStatement(LOG_CTX, ps);
}
}
/**
* This method is called to perform 'updates' for any inserts that failed.
*
* @return The data insert result containing the data points that were not
* updated.
*/
private List<DataPoint> updateData(Connection conn, List<DataPoint> data) {
PreparedStatement stmt = null;
List<DataPoint> left = new ArrayList<DataPoint>();
Map<String, List<DataPoint>> buckets = MeasRangeObj.getInstance().bucketData(data);
for (Entry<String, List<DataPoint>> entry : buckets.entrySet()) {
String table = entry.getKey();
List<DataPoint> dpts = entry.getValue();
try {
// TODO need to set synchronous commit to off
stmt = conn.prepareStatement("UPDATE " + table +
" SET value = ? WHERE timestamp = ? AND measurement_id = ?");
for (DataPoint pt : dpts) {
Integer metricId = pt.getMeasurementId();
MetricValue val = pt.getMetricValue();
BigDecimal bigDec;
bigDec = new BigDecimal(val.getValue());
stmt.setBigDecimal(1, getDecimalInRange(bigDec, metricId));
stmt.setLong(2, val.getTimestamp());
stmt.setInt(3, metricId.intValue());
stmt.addBatch();
}
int[] execInfo = stmt.executeBatch();
left.addAll(getRemainingDataPoints(dpts, execInfo));
} catch (BatchUpdateException e) {
left.addAll(getRemainingDataPointsAfterBatchFail(dpts, e.getUpdateCounts()));
} catch (SQLException e) {
// If the batch update is not within a transaction, then we
// don't know which of the updates completed successfully.
// Assume they all failed.
left.addAll(dpts);
if (log.isDebugEnabled()) {
log.debug("A general SQLException occurred during the update. " +
"Assuming that none of the " + dpts.size() +
" data points were updated.", e);
}
} finally {
DBUtil.closeStatement(LOG_CTX, stmt);
}
}
return left;
}
/**
* Get the server purge configuration and storage option, loaded on startup.
*/
private void loadConfigDefaults() {
log.debug("Loading default purge intervals");
Properties conf;
try {
conf = serverConfigManager.getConfig();
} catch (Exception e) {
throw new SystemException(e);
}
String purgeRawString = conf.getProperty(HQConstants.DataPurgeRaw);
String purge1hString = conf.getProperty(HQConstants.DataPurge1Hour);
String purge6hString = conf.getProperty(HQConstants.DataPurge6Hour);
String purge1dString = conf.getProperty(HQConstants.DataPurge1Day);
try {
purgeRaw = Long.parseLong(purgeRawString);
purge1h = Long.parseLong(purge1hString);
purge6h = Long.parseLong(purge6hString);
purge1d = Long.parseLong(purge1dString);
confDefaultsLoaded = true;
} catch (NumberFormatException e) {
// Shouldn't happen unless manual edit of config table
throw new IllegalArgumentException("Invalid purge interval: " + e);
}
}
/**
* find the aggregation table with the greatest interval which is smaller than the requested time frame
* and which the time frame doesn't cover a range which is even partially after its purge has been done
* and in which no more than maxDTPs fit in the requested time frame
* @throws TimeframeSizeException
* @throws TimeframeBoundriesException
*/
protected String getDataTable(long begin, long end, Measurement msmt, int maxDTPs) throws TimeframeSizeException, TimeframeBoundriesException {
long tf = end - begin; // the time frame
long maxInterval = tf / maxDTPs; // the max interval for which maxDTPs DTPs still fit in the time frame
long msmtInterval = msmt.getInterval();
if (tf<msmtInterval) {
MeasurementTemplate tmp = msmt.getTemplate();
throw new TimeframeSizeException("The requested time frame is of size " + tf + " milliseconds, which is smaller than the time interval of the measurement" + (tmp!=null?(" "+tmp.getName()):"") + " which is " + msmtInterval + " milliseconds");
}
long now = System.currentTimeMillis();
if ((end>now) || (end<begin)) {
throw new TimeframeBoundriesException("The requested time frame"
+ (end>now?" ends in the future":(end<begin?" and":""))
+ (end<begin?" ends before it starts":""));
}
if (!confDefaultsLoaded) {
loadConfigDefaults();
}
if ((msmtInterval>=maxInterval) && (begin>=(now-DataManagerImpl.this.purgeRaw))) {
return MeasurementUnionStatementBuilder.getUnionStatement(begin, end, msmt.getId().intValue(), measurementDAO.getHQDialect());
}
if (tf<HOUR_IN_MILLI) {
MeasurementTemplate tmp = msmt.getTemplate();
throw new TimeframeSizeException("The requested time frame is of size " + tf + " milliseconds, which is smaller than the hourly aggregated time interval of the measurement " + (tmp!=null?tmp.getName():""));
}
if ((HOUR_IN_MILLI>=maxInterval) && (begin>=(now-DataManagerImpl.this.purge1h))) {
return MeasurementConstants.TAB_DATA_1H;
}
if (tf<SIX_HOURS_IN_MILLI) {
MeasurementTemplate tmp = msmt.getTemplate();
throw new TimeframeSizeException("The requested time frame is of size " + tf + " milliseconds, which is smaller than the 6-hourly aggregated time interval of the measurement " + (tmp!=null?tmp.getName():""));
}
if ((SIX_HOURS_IN_MILLI>=maxInterval) && (begin>=(now-DataManagerImpl.this.purge6h))) {
return MeasurementConstants.TAB_DATA_6H;
}
if (tf<DAY_IN_MILLI) {
MeasurementTemplate tmp = msmt.getTemplate();
throw new TimeframeSizeException("The requested time frame is of size " + tf + " milliseconds, which is smaller than the daily aggregated time interval of the measurement " + (tmp!=null?tmp.getName():""));
}
// return daily aggregated data even if the time frame is beyond the purge time or contains more than 400 DTPs
return MeasurementConstants.TAB_DATA_1D;
}
public String getDataTable(long begin, long end, int measId) {
Integer[] empty = new Integer[1];
empty[0] = new Integer(measId);
return getDataTable(begin, end, empty);
}
private boolean usesMetricUnion(long begin) {
long now = System.currentTimeMillis();
if (MeasTabManagerUtil.getMeasTabStartTime(now - getPurgeRaw()) < begin) {
return true;
}
return false;
}
private boolean usesMetricUnion(long begin, long end, boolean useAggressiveRollup) {
if ((!useAggressiveRollup && usesMetricUnion(begin)) ||
(useAggressiveRollup && (((end - begin) / HOUR) < HOURS_PER_MEAS_TAB))) {
return true;
}
return false;
}
private String getDataTable(long begin, long end, Integer[] measIds) {
return getDataTable(begin, end, measIds, false);
}
/**
* @param begin beginning of the time range
* @param end end of the time range
* @param useAggressiveRollup will use the rollup tables if
* the timerange represents the same timerange as one
* metric data table
*/
private String[] getDataTables(long begin, long end, boolean useAggressiveRollup) {
long now = System.currentTimeMillis();
if (!confDefaultsLoaded) {
loadConfigDefaults();
}
if (usesMetricUnion(begin, end, useAggressiveRollup)) {
return MeasTabManagerUtil.getMetricTables(begin, end);
} else if ((now - this.purge1h) < begin) {
return new String[] { TAB_DATA_1H };
} else if ((now - this.purge6h) < begin) {
return new String[] { TAB_DATA_6H };
} else {
return new String[] { TAB_DATA_1D };
}
}
/**
* @param begin beginning of the time range
* @param end end of the time range
* @param measIds the measurement_ids associated with the query. This is
* only used for 'UNION ALL' queries
* @param useAggressiveRollup will use the rollup tables if the timerange
* represents the same timerange as one metric data table
*/
private String getDataTable(long begin, long end, Integer[] measIds, boolean useAggressiveRollup) {
long now = System.currentTimeMillis();
if (!confDefaultsLoaded) {
loadConfigDefaults();
}
if (usesMetricUnion(begin, end, useAggressiveRollup)) {
return MeasurementUnionStatementBuilder.getUnionStatement(begin, end, measIds,
measurementDAO.getHQDialect());
} else if ((now - this.purge1h) < begin) {
return TAB_DATA_1H;
} else if ((now - this.purge6h) < begin) {
return TAB_DATA_6H;
} else {
return TAB_DATA_1D;
}
}
@Transactional(readOnly = true)
public List<HighLowMetricValue> getHistoricalData(Measurement m, long begin, long end,
boolean prependAvailUnknowns, int maxDTPs) throws TimeframeSizeException, TimeframeBoundriesException {
if (m.getTemplate().isAvailability()) {
return availabilityManager.getHistoricalAvailData(m, begin, end, PageControl.PAGE_ALL, prependAvailUnknowns);
} else {
return getNonAvailabilityMetricData(m, begin, end, maxDTPs);
}
}
/**
* Fetch the list of historical data points given a begin and end time
* range. Returns a PageList of DataPoints without begin rolled into time
* windows.
*
* @param m The Measurement
* @param begin the start of the time range
* @param end the end of the time range
* @param prependAvailUnknowns determines whether to prepend AVAIL_UNKNOWN if the
* corresponding time window is not accounted for in the database.
* Since availability is contiguous this will not occur unless the
* time range precedes the first availability point.
* @return the list of data points
*
*/
@Transactional(readOnly = true)
public PageList<HighLowMetricValue> getHistoricalData(Measurement m, long begin, long end,
PageControl pc,
boolean prependAvailUnknowns) {
if (m.getTemplate().isAvailability()) {
return availabilityManager.getHistoricalAvailData(m, begin, end, pc,
prependAvailUnknowns);
} else {
return getHistData(m, begin, end, pc);
}
}
/**
* Fetch the list of historical data points given a begin and end time
* range. Returns a PageList of DataPoints without begin rolled into time
* windows.
*
* @param m The Measurement
* @param begin the start of the time range
* @param end the end of the time range
* @return the list of data points
*
*/
@Transactional(readOnly = true)
public PageList<HighLowMetricValue> getHistoricalData(Measurement m, long begin, long end,
PageControl pc) {
return getHistoricalData(m, begin, end, pc, false);
}
private PageList<HighLowMetricValue> getHistData(final Measurement m, long begin, long end,
final PageControl pc) {
checkTimeArguments(begin, end);
begin = TimingVoodoo.roundDownTime(begin, MINUTE);
end = TimingVoodoo.roundDownTime(end, MINUTE);
final ArrayList<HighLowMetricValue> history = new ArrayList<HighLowMetricValue>();
// Get the data points and add to the ArrayList
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
// The table to query from
final String table = getDataTable(begin, end, m.getId().intValue());
final HQDialect dialect = measurementDAO.getHQDialect();
try {
conn = dbUtil.getConnection();
stmt = conn.createStatement();
try {
final boolean sizeLimit = (pc.getPagesize() != PageControl.SIZE_UNLIMITED);
final StringBuilder sqlBuf = new StringBuilder()
.append("SELECT :fields FROM ")
.append(table).append(" WHERE timestamp BETWEEN ")
.append(begin).append(" AND ").append(end)
.append(" AND measurement_id=").append(m.getId())
.append(" ORDER BY timestamp ").append(pc.isAscending() ? "ASC" : "DESC");
Integer total = null;
if (sizeLimit) {
// need to get the total count if there is a limit on the
// size. Otherwise we can just take the size of the
// resultset
rs = stmt.executeQuery(sqlBuf.toString().replace(":fields", "count(*)"));
total = (rs.next()) ? new Integer(rs.getInt(1)) : new Integer(1);
rs.close();
if (total.intValue() == 0) {
return new PageList<HighLowMetricValue>();
}
if (log.isDebugEnabled()) {
log.debug("paging: offset= " + pc.getPageEntityIndex() + ", pagesize=" +
pc.getPagesize());
}
}
final int offset = pc.getPageEntityIndex();
final int limit = pc.getPagesize();
final String sql = (sizeLimit) ?
dialect.getLimitBuf(sqlBuf.toString(), offset, limit) : sqlBuf.toString();
if (log.isDebugEnabled()) {
log.debug(sql);
}
final StopWatch timer = new StopWatch();
rs = stmt.executeQuery(sql.replace(":fields", "value, timestamp"));
if (log.isTraceEnabled()) {
log.trace("getHistoricalData() execute time: " + timer.getElapsed());
}
while (rs.next()) {
history.add(getMetricValue(rs));
}
total = (total == null) ? new Integer(history.size()) : total;
return new PageList<HighLowMetricValue>(history, total.intValue());
} catch (SQLException e) {
throw new SystemException("Can't lookup historical data for " + m, e);
}
} catch (SQLException e) {
throw new SystemException("Can't open connection", e);
} finally {
DBUtil.closeJDBCObjects(LOG_CTX, conn, stmt, rs);
}
}
private List<HighLowMetricValue> getNonAvailabilityMetricData(final Measurement m, long begin, long end, int maxDTPs) throws TimeframeSizeException, TimeframeBoundriesException {
checkTimeArguments(begin, end);
begin = TimingVoodoo.roundDownTime(begin, MINUTE);
end = TimingVoodoo.roundDownTime(end, MINUTE);
final ArrayList<HighLowMetricValue> history = new ArrayList<HighLowMetricValue>();
// Get the data points and add to the ArrayList
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
// The table to query from
final String table = getDataTable(begin, end, m,maxDTPs);
try {
conn = dbUtil.getConnection();
stmt = conn.createStatement();
try {
final StringBuilder sqlBuf = new StringBuilder()
.append("SELECT ")
.append((table.equals(TAB_DATA_1H) || table.equals(TAB_DATA_6H) || table.equals(TAB_DATA_1D))?
("maxvalue as peak, minvalue as low, "):"")
.append("value, timestamp FROM ")
.append(table).append(" WHERE timestamp BETWEEN ")
.append(begin).append(" AND ").append(end)
.append(" AND measurement_id=").append(m.getId())
.append(" ORDER BY timestamp ").append("ASC");
final String sql = sqlBuf.toString();
if (log.isDebugEnabled()) {
log.debug(sql);
}
rs = stmt.executeQuery(sql);
while (rs.next()) {
history.add(getMetricValue(rs));
}
return new ArrayList<HighLowMetricValue>(history);
} catch (SQLException e) {
throw new SystemException("Can't lookup historical data for " + m, e);
}
} catch (SQLException e) {
throw new SystemException("Can't open connection", e);
} finally {
DBUtil.closeJDBCObjects(LOG_CTX, conn, stmt, rs);
}
}
public Collection<HighLowMetricValue> getRawData(Measurement m, long begin, long end,
AtomicLong publishedInterval) {
final long interval = m.getInterval();
begin = TimingVoodoo.roundDownTime(begin, interval);
end = TimingVoodoo.roundDownTime(end, interval);
Collection<HighLowMetricValue> points;
if (m.getTemplate().isAvailability()) {
points = availabilityManager.getHistoricalAvailData(
new Integer[] { m.getId() }, begin, end, interval, PageControl.PAGE_ALL, true);
publishedInterval.set(interval);
} else {
points = getRawDataPoints(m, begin, end, publishedInterval);
}
return points;
}
private TreeSet<HighLowMetricValue> getRawDataPoints(Measurement m, long begin, long end,
AtomicLong publishedInterval) {
final StringBuilder sqlBuf = getRawDataSql(m, begin, end, publishedInterval);
final TreeSet<HighLowMetricValue> rtn =
new TreeSet<HighLowMetricValue>(getTimestampComparator());
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try {
conn = safeGetConnection();
stmt = conn.createStatement();
rs = stmt.executeQuery(sqlBuf.toString());
final int valCol = rs.findColumn("value");
final int timestampCol = rs.findColumn("timestamp");
while (rs.next()) {
final double val = rs.getDouble(valCol);
final long timestamp = rs.getLong(timestampCol);
rtn.add(new HighLowMetricValue(val, timestamp));
}
} catch (SQLException e) {
throw new SystemException(e);
} finally {
DBUtil.closeJDBCObjects(LOG_CTX, conn, stmt, rs);
}
return rtn;
}
private StringBuilder getRawDataSql(Measurement m, long begin, long end,
AtomicLong publishedInterval) {
final String sql = new StringBuilder(128).append("SELECT value, timestamp FROM :table")
.append(" WHERE timestamp BETWEEN ").append(begin).append(" AND ").append(end).append(
" AND measurement_id=").append(m.getId()).toString();
final String[] tables = getDataTables(begin, end, false);
if (tables.length == 1) {
if (tables[0].equals(TAB_DATA_1H)) {
publishedInterval.set(HOUR);
} else if (tables[0].equals(TAB_DATA_6H)) {
publishedInterval.set(HOUR * 6);
} else if (tables[0].equals(TAB_DATA_1D)) {
publishedInterval.set(HOUR * 24);
}
}
final StringBuilder sqlBuf = new StringBuilder(128 * tables.length);
for (int i = 0; i < tables.length; i++) {
sqlBuf.append(sql.replace(":table", tables[i]));
if (i < (tables.length - 1)) {
sqlBuf.append(" UNION ALL ");
}
}
return sqlBuf;
}
private Comparator<MetricValue> getTimestampComparator() {
return new Comparator<MetricValue>() {
public int compare(MetricValue arg0, MetricValue arg1) {
Long point0 = arg0.getTimestamp();
Long point1 = arg1.getTimestamp();
return point0.compareTo(point1);
}
};
}
/**
* Aggregate data across the given metric IDs, returning max, min, avg, and
* count of number of unique metric IDs
*
* @param measurements {@link List} of {@link Measurement}s
* @param begin The start of the time range
* @param end The end of the time range
* @return the An array of aggregate values
*
*/
@Transactional(readOnly = true)
public double[] getAggregateData(final List<Measurement> measurements, final long begin,
final long end) {
checkTimeArguments(begin, end);
long interval = end - begin;
interval = (interval == 0) ? 1 : interval;
final List<HighLowMetricValue> pts = getHistoricalData(measurements, begin, end, interval,
MeasurementConstants.COLL_TYPE_DYNAMIC, false, PageControl.PAGE_ALL);
return getAggData(pts);
}
@Transactional(readOnly = true)
public Map<Integer, double[]> getAggregateDataAndAvailUpByMetric(final List<Measurement> measurements,
final long begin, final long end) throws SQLException {
List<Integer> avids = new ArrayList<Integer>();
List<Integer> mids = new ArrayList<Integer>();
for (Measurement meas : measurements) {
MeasurementTemplate t = meas.getTemplate();
if (t.isAvailability()) {
avids.add(meas.getId());
} else {
mids.add(meas.getId());
}
}
Map<Integer, double[]> rtn = getAggDataByMetric(mids.toArray(new Integer[0]),begin, end, false);
rtn.putAll(availabilityManager.getAggregateDataAndAvailUpByMetric(avids,begin, end));
return rtn;
}
/**
* Fetch the list of historical data points, grouped by template, given a
* begin and end time range. Does not return an entry for templates with no
* associated data. PLEASE NOTE: The
* {@link MeasurementConstants.IND_LAST_TIME} index in the {@link double[]}
* part of the returned map does not contain the real last value. Instead it
* is an averaged value calculated from the last 1/60 of the specified time
* range. If this becomes an issue the best way I can think of to solve is
* to pass in a boolean "getRealLastTime" and issue another query to get the
* last value if this is set. It is much better than the alternative of
* always querying the last metric time because not all pages require this
* value.
*
* @param measurements The List of {@link Measurement}s to query
* @param begin The start of the time range
* @param end The end of the time range
* @see org.hyperic.hq.measurement.server.session.AvailabilityManagerImpl#getHistoricalData()
* @return the {@link Map} of {@link Integer} to {@link double[]} which
* represents templateId to data points
*
*/
@Transactional(readOnly = true)
public Map<Integer, double[]> getAggregateDataByTemplate(final List<Measurement> measurements,
final long begin, final long end) {
// the idea here is to try and match the exact query executed by
// getHistoricalData() when viewing the metric indicators page.
// By issuing the same query we are hoping that the db's query
// cache will optimize performance.
checkTimeArguments(begin, end);
final long interval = (end - begin) / 60;
final List<Integer> availIds = new ArrayList<Integer>();
final Map<Integer, List<Measurement>> measIdsByTempl = new HashMap<Integer, List<Measurement>>();
setMeasurementObjects(measurements, availIds, measIdsByTempl);
final Integer[] avIds = availIds.toArray(new Integer[0]);
final Map<Integer, double[]> rtn = availabilityManager.getAggregateDataByTemplate(avIds,
begin, end);
rtn.putAll(getAggDataByTempl(measIdsByTempl, begin, end, interval));
return rtn;
}
private Map<Integer, double[]> getAggDataByTempl(final Map<Integer,List<Measurement>> measIdsByTempl,
final long begin,
final long end,
final long interval) {
final HashMap<Integer, double[]> rtn = new HashMap<Integer, double[]>(measIdsByTempl.size());
for (final Map.Entry<Integer, List<Measurement>> entry : measIdsByTempl.entrySet()) {
final Integer tid = entry.getKey();
final List<Measurement> meas = entry.getValue();
final List<HighLowMetricValue> pts = getHistoricalData(meas, begin, end, interval,
MeasurementConstants.COLL_TYPE_DYNAMIC, false, PageControl.PAGE_ALL);
final double[] aggData = getAggData(pts);
if (aggData == null) {
continue;
}
rtn.put(tid, aggData);
}
return rtn;
}
/**
* @param measurements source measurements list
* @param availIds measurements from the source list of type availability
* @param measIdsByTempl measurements from the source list which are not of type availability, sorted as per their type
*/
private final void setMeasurementObjects(final List<Measurement> measurements,
final List<Integer> availIds,
final Map<Integer, List<Measurement>> measIdsByTempl) {
for (Measurement m : measurements) {
final MeasurementTemplate t = m.getTemplate();
if (m.getTemplate().isAvailability()) {
availIds.add(m.getId());
} else {
final Integer tid = t.getId();
List<Measurement> list;
if (null == (list = measIdsByTempl.get(tid))) {
list = new ArrayList<Measurement>();
measIdsByTempl.put(tid, list);
}
list.add(m);
}
}
}
private final double[] getAggData(final List<HighLowMetricValue> historicalData) {
if (historicalData.size() == 0) {
return null;
}
double high = Double.MIN_VALUE;
double low = Double.MAX_VALUE;
double total = 0;
Double lastVal = null;
int count = 0;
long last = Long.MIN_VALUE;
for (HighLowMetricValue mv : historicalData) {
low = Math.min(mv.getLowValue(), low);
high = Math.max(mv.getHighValue(), high);
if (mv.getTimestamp() > last) {
lastVal = new Double(mv.getValue());
}
final int c = mv.getCount();
count = count + c;
total = ((mv.getValue() * c) + total);
}
final double[] data = new double[MeasurementConstants.IND_LAST_TIME + 1];
data[MeasurementConstants.IND_MIN] = low;
data[MeasurementConstants.IND_AVG] = total / count;
data[MeasurementConstants.IND_MAX] = high;
data[MeasurementConstants.IND_CFG_COUNT] = count;
if (lastVal != null) {
data[MeasurementConstants.IND_LAST_TIME] = lastVal.doubleValue();
}
return data;
}
/**
* Fetch the list of historical data points given a start and stop time
* range and interval
*
* @param measurements The List of Measurements to query
* @param begin The start of the time range
* @param end The end of the time range
* @param interval Interval for the time range
* @param type Collection type for the metric
* @param returnMetricNulls Specifies whether intervals with no data should
* be return as {@link HighLowMetricValue} with the value set as
* Double.NaN
* @see org.hyperic.hq.measurement.server.session.AvailabilityManagerImpl#getHistoricalData()
* @return the list of data points
*
*/
@Transactional(readOnly = true)
public PageList<HighLowMetricValue> getHistoricalData(final List<Measurement> measurements,
long begin, long end, long interval,
int type, boolean returnMetricNulls,
PageControl pc) {
begin = TimingVoodoo.roundDownTime(begin, MINUTE);
end = TimingVoodoo.roundUpTime(end, MINUTE);
final List<Integer> availIds = new ArrayList<Integer>();
final List<Integer> measIds = new ArrayList<Integer>();
checkTimeArguments(begin, end, interval);
interval = (interval == 0) ? 1 : interval;
for (final Measurement m : measurements) {
if (m.getTemplate().isAvailability()) {
availIds.add(m.getId());
} else {
measIds.add(m.getId());
}
}
final Integer[] avIds = availIds.toArray(new Integer[0]);
final PageList<HighLowMetricValue> rtn =
availabilityManager.getHistoricalAvailData(avIds, begin, end, interval, pc, true);
final HQDialect dialect = measurementDAO.getHQDialect();
final int points = (int) ((begin-end)/interval);
final int maxExprs = (dialect.getMaxExpressions() == -1) ?
Integer.MAX_VALUE : dialect.getMaxExpressions();
for (int i = 0; i < measIds.size(); i += maxExprs) {
final int last = Math.min(i + maxExprs, measIds.size());
final List<Integer> sublist = measIds.subList(i, last);
final Integer[] mids = sublist.toArray(new Integer[0]);
final Collection<HighLowMetricValue> coll =
getHistData(mids, begin, end, interval, returnMetricNulls, null);
final PageList<HighLowMetricValue> pList =
new PageList<HighLowMetricValue>(coll, points);
merge(rtn, pList);
}
return rtn;
}
private void merge(int bucket, AggMetricValue[] rtn, AggMetricValue val, long timestamp) {
AggMetricValue amv = null;
try {
amv = rtn[bucket];
if (amv == null) {
rtn[bucket] = val;
} else {
amv.merge(val);
}
} catch (NullPointerException e) {
log.error("Error has occured in Merge function, bucket size[" + bucket + "] " +
"but AggMetricValue array is null, " + e, e);
} catch (ArrayIndexOutOfBoundsException e){
log.error("Error has occured in Merge function, bucket size[" + bucket + "] " +
"but AggMetricValue array size is [" + rtn.length +"] ," + e, e);
}
}
private Collection<HighLowMetricValue> getHistData(Integer[] mids, long start, long finish,
long windowSize, final boolean returnNulls,
AtomicLong publishedInterval) {
final int buckets = (int) ((finish - start) / windowSize);
final Collection<HighLowMetricValue> rtn = new ArrayList<HighLowMetricValue>(buckets);
long tmp = start;
final AggMetricValue[] values = getAggValueSets(mids, start, finish,
windowSize, returnNulls, publishedInterval);
for (int ii = 0; ii < values.length; ii++) {
final AggMetricValue val = values[ii];
if ((null == val) && returnNulls) {
rtn.add(new HighLowMetricValue(Double.NaN, tmp + (ii * windowSize)));
continue;
} else if (null == val) {
continue;
} else {
rtn.add(val.getHighLowMetricValue());
}
}
return rtn;
}
private class AggMetricValue {
private int count;
private double max;
private double min;
private double sum;
private final long timestamp;
private AggMetricValue(long timestamp, double sum, double max, double min, int count) {
this.timestamp = timestamp;
this.sum = sum;
this.max = max;
this.min = min;
this.count = count;
}
private void merge(AggMetricValue val) {
count += val.count;
max = (val.max > max) ? val.max : max;
min = (val.min < min) ? val.min : min;
sum += val.sum;
}
@SuppressWarnings("unused")
private void set(double val) {
count++;
max = (val > max) ? val : max;
min = (val < min) ? val : min;
sum += val;
}
private double getAvg() {
return sum/count;
}
private HighLowMetricValue getHighLowMetricValue() {
HighLowMetricValue rtn = new HighLowMetricValue(getAvg(), max, min, timestamp);
rtn.setCount(count);
return rtn;
}
}
private CharSequence getRawDataSql(Integer[] mids, long begin, long end,
AtomicLong publishedInterval) {
if ((mids == null) || (mids.length == 0)) {
return "";
}
if (log.isDebugEnabled()) {
log.debug("gathering data from begin=" + TimeUtil.toString(begin) +
", end=" + TimeUtil.toString(end));
}
final HQDialect dialect = measurementDAO.getHQDialect();
// XXX I don't like adding the sql hint, when we start testing against mysql 5.5 we should
// re-evaluate if this is necessary
// 1) we shouldn't have to tell the db to explicitly use the Primary key for these
// queries, it should just know because we update stats every few hours
// 2) we only want to use the primary key for bigger queries. Our tests show
// that the primary key performance is very consistent for large queries and smaller
// queries. But for smaller queries the measurement_id index is more effective
final String hint = (dialect.getMetricDataHint().isEmpty() || (mids.length < 1000)) ?
"" : " " + dialect.getMetricDataHint();
final String sql = new StringBuilder(1024 + (mids.length * 5))
.append("SELECT count(*) as cnt, sum(value) as sumvalue, ")
.append("min(value) as minvalue, max(value) as maxvalue, timestamp")
.append(" FROM :table").append(hint)
.append(" WHERE timestamp BETWEEN ").append(begin).append(" AND ").append(end)
.append(MeasTabManagerUtil.getMeasInStmt(mids, true))
.append(" GROUP BY timestamp")
.toString();
final String[] tables = getDataTables(begin, end, false);
if ((publishedInterval != null) && (tables.length == 1)) {
if (tables[0].equals(TAB_DATA_1H)) {
publishedInterval.set(HOUR);
} else if (tables[0].equals(TAB_DATA_6H)) {
publishedInterval.set(HOUR * 6);
} else if (tables[0].equals(TAB_DATA_1D)) {
publishedInterval.set(HOUR * 24);
}
}
final StringBuilder sqlBuf = new StringBuilder(128 * tables.length);
for (int i = 0; i < tables.length; i++) {
sqlBuf.append(sql.replace(":table", tables[i]));
if (i < (tables.length - 1)) {
sqlBuf.append(" UNION ALL ");
}
}
return sqlBuf;
}
private AggMetricValue[] getAggValueSets(final Integer[] mids,
final long start, final long finish,
final long windowSize, final boolean returnNulls,
final AtomicLong publishedInterval) {
final String[] tables = getDataTables(start, finish, false);
if (tables.length <= 0) {
throw new SystemException(
"ERROR: no data tables represent range " + TimeUtil.toString(start) +
" - " + TimeUtil.toString(finish));
}
final MeasRange[] ranges = (tables.length > 1) ?
MeasTabManagerUtil.getMetricRanges(start, finish) :
new MeasRange[] {new MeasRange(tables[0], start, finish)};
final String threadName = Thread.currentThread().getName();
final List<Thread> threads = new ArrayList<Thread>(ranges.length);
final Collection<AggMetricValue[]> data = new ArrayList<AggMetricValue[]>(ranges.length);
final int maxThreads = 4;
// The result encapsulates the timeframe start -> finish. The results are gathered
// via sub-queries. Each sub-query is from begin -> end
// start finish
// <----------------------------------------------------------------------------->
// (begin-end)(begin-end)(begin-end)(begin-end)(begin-end)(begin-end)(begin-end)..
for (final MeasRange range : ranges) {
final long min = range.getMinTimestamp();
final long max = range.getMaxTimestamp();
final long begin = (min < start) ? start : min;
final long end = (max > finish) ? finish : max;
waitForThreads(threads, maxThreads);
// XXX may want to add a thread pool or a static Executor here so that these
// queries don't overwhelm the DB
Thread thread = getNewDataWorkerThread(mids, start, finish, begin, end, windowSize,
returnNulls, publishedInterval, threadName,
data);
thread.start();
threads.add(thread);
}
waitForThreads(threads);
return mergeThreadData(start, finish, windowSize, data);
}
/**
* @param begin - the begin time of the sub window
* @param end - the end time of the sub window
* @param start - the start time of the user specified window
* @param finish - the finish time of the user specified window
*/
private Thread getNewDataWorkerThread(final Integer[] mids, final long start,
final long finish, final long begin, final long end,
final long windowSize, final boolean returnNulls,
final AtomicLong publishedInterval,
final String threadName,
final Collection<AggMetricValue[]> data) {
final boolean debug = log.isDebugEnabled();
final Thread thread = new Thread() {
@Override
public void run() {
final StopWatch watch = new StopWatch();
if (debug) {
watch.markTimeBegin("data gatherer begin=" + TimeUtil.toString(begin) +
", end=" + TimeUtil.toString(end));
}
final AggMetricValue[] array = getHistDataSet(mids, start, finish, begin, end,
windowSize, returnNulls,
publishedInterval, threadName);
if (debug) {
watch.markTimeEnd("data gatherer begin=" + TimeUtil.toString(begin) +
", end=" + TimeUtil.toString(end));
log.debug(watch);
}
synchronized (data) {
data.add(array);
}
}
};
return thread;
}
private AggMetricValue[] getHistDataSet(Integer[] mids, long start, long finish,
long rangeBegin, long rangeEnd,
long windowSize, final boolean returnNulls,
AtomicLong publishedInterval, String threadName) {
final CharSequence sqlBuf = getRawDataSql(mids, rangeBegin, rangeEnd, publishedInterval);
final int buckets = (int) ((finish - start) / windowSize);
final AggMetricValue[] array = new AggMetricValue[buckets];
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try {
conn = safeGetConnection();
stmt = conn.createStatement();
int timeout = stmt.getQueryTimeout();
if (timeout == 0) {
stmt.setQueryTimeout(transactionTimeout);
}
rs = stmt.executeQuery("/* " + threadName + " */ " + sqlBuf.toString());
final int sumValCol = rs.findColumn("sumvalue");
final int countValCol = rs.findColumn("cnt");
final int minValCol = rs.findColumn("minvalue");
final int maxValCol = rs.findColumn("maxvalue");
final int timestampCol = rs.findColumn("timestamp");
while (rs.next()) {
final double sum = rs.getDouble(sumValCol);
final double min = rs.getDouble(minValCol);
final double max = rs.getDouble(maxValCol);
final int count = rs.getInt(countValCol);
final long timestamp = rs.getLong(timestampCol);
final AggMetricValue val = new AggMetricValue(timestamp, sum,
max, min, count);
if ((timestamp < start) || (timestamp > finish)) {
continue;
}
final int bucket = (int) (buckets - ((finish - timestamp) / (float) windowSize));
if (bucket < 0) {
continue;
}
merge(bucket, array, val, timestamp);
}
} catch (SQLException e) {
throw new SystemException(e);
} finally {
DBUtil.closeJDBCObjects(getClass().getName(), conn, stmt, rs);
}
return array;
}
private AggMetricValue[] mergeThreadData(long start, long finish, long windowSize,
Collection<AggMetricValue[]> data) {
final int buckets = (int) ((finish - start) / windowSize);
final AggMetricValue[] rtn = new AggMetricValue[buckets];
for (final AggMetricValue[] vals : data) {
for (int ii = 0; ii < vals.length; ii++) {
final AggMetricValue val = vals[ii];
if (val == null) {
continue;
}
if (rtn[ii] == null) {
rtn[ii] = val;
} else {
rtn[ii].merge(val);
}
}
}
return rtn;
}
private void waitForThreads(List<Thread> threads) {
for (final Thread thread : threads) {
while (thread.isAlive()) {
try {
thread.join();
} catch (InterruptedException e) {
log.debug(e, e);
}
}
}
}
private void waitForThreads(List<Thread> threads, int maxThreads) {
if (threads.isEmpty() || (threads.size() < maxThreads)) {
return;
}
int i=0;
while (threads.size() >= maxThreads) {
i = ((i >= threads.size()) || (i < 0)) ? 0 : i;
final Thread thread = threads.get(i);
try {
if (thread.isAlive()) {
thread.join(100);
}
if (!thread.isAlive()) {
threads.remove(i);
} else {
i++;
}
} catch (InterruptedException e) {
log.debug(e,e);
}
}
}
private void merge(PageList<HighLowMetricValue> master, PageList<HighLowMetricValue> toMerge) {
if (master.size() == 0) {
master.addAll(toMerge);
return;
}
for (int i = 0; i < master.size(); i++) {
if (toMerge.size() < (i + 1)) {
break;
}
final HighLowMetricValue val = master.get(i);
final HighLowMetricValue mval = toMerge.get(i);
final int mcount = mval.getCount();
final int count = val.getCount();
final int tot = count + mcount;
final double high = ((val.getHighValue() * count) + (mval.getHighValue() * mcount)) /
tot;
val.setHighValue(high);
final double low = ((val.getLowValue() * count) + (mval.getLowValue() * mcount)) / tot;
val.setLowValue(low);
final double value = ((val.getValue() * count) + (mval.getValue() * mcount)) / tot;
val.setValue(value);
}
}
private long getPurgeRaw() {
if (!confDefaultsLoaded) {
loadConfigDefaults();
}
return purgeRaw;
}
/**
*
* Get the last MetricValue for the given Measurement.
*
* @param m The Measurement
* @return The MetricValue or null if one does not exist.
*
*/
@Transactional(readOnly = true)
public MetricValue getLastHistoricalData(Measurement m) {
if (m.getTemplate().isAvailability()) {
return availabilityManager.getLastAvail(m);
} else {
return getLastHistData(m);
}
}
private MetricValue getLastHistData(Measurement m) {
// Check the cache
MetricValue mval = metricDataCache.get(m.getId(), 0);
if (mval != null) {
return mval;
}
// Get the data points and add to the ArrayList
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try {
conn = dbUtil.getConnection();
final String metricUnion =
MeasurementUnionStatementBuilder.getUnionStatement(8 * HOUR, m.getId(), measurementDAO.getHQDialect());
final StringBuilder sqlBuf = new StringBuilder()
.append("SELECT timestamp, value FROM ").append(metricUnion)
.append(", (SELECT MAX(timestamp) AS maxt").append(" FROM ").append(metricUnion)
.append(") mt ")
.append("WHERE measurement_id = ").append(m.getId()).append(" AND timestamp = maxt");
stmt = conn.createStatement();
if (log.isDebugEnabled()) {
log.debug("getLastHistoricalData(): " + sqlBuf);
}
rs = stmt.executeQuery(sqlBuf.toString());
if (rs.next()) {
MetricValue mv = getMetricValue(rs);
metricDataCache.add(m.getId(), mv);
return mv;
} else {
// No cached value, nothing in the database
return null;
}
} catch (SQLException e) {
log.error("Unable to look up historical data for " + m, e);
throw new SystemException(e);
} finally {
DBUtil.closeJDBCObjects(LOG_CTX, conn, stmt, rs);
}
}
/**
* Fetch the most recent data point for particular Measurements.
*
* @param mids The List of Measurements to query. In the list of
* Measurements null values are allowed as placeholders.
* @param timestamp Only use data points with collection times greater than
* the given timestamp.
* @return A Map of measurement ids to MetricValues. TODO: We should change
* this method to now allow NULL values. This is legacy and only
* used by the Metric viewer and Availabilty Summary portlets.
*
*/
@Transactional(readOnly = true)
public Map<Integer, MetricValue> getLastDataPoints(List<Integer> mids, long timestamp) {
final List<Integer> availIds = new ArrayList<Integer>(mids.size());
final List<Integer> measurementIds = new ArrayList<Integer>(mids.size());
for (Integer measId : mids) {
if (measId == null) {
// See above.
measurementIds.add(null);
continue;
}
final Measurement m = measurementDAO.get(measId);
if (m == null) {
// See above.
measurementIds.add(null);
continue;
} else if (m.getTemplate().isAvailability()) {
availIds.add(m.getId());
} else {
measurementIds.add(measId);
}
}
final Integer[] avIds = availIds.toArray(new Integer[0]);
final StopWatch watch = new StopWatch();
final boolean debug = log.isDebugEnabled();
if (debug) {
watch.markTimeBegin("getLastDataPts");
}
final Map<Integer, MetricValue> data = getLastDataPts(measurementIds, timestamp);
if (debug) {
watch.markTimeEnd("getLastDataPts");
}
if (availIds.size() > 0) {
if (debug) {
watch.markTimeBegin("getLastAvail");
}
data.putAll(availabilityManager.getLastAvail(avIds));
if (debug) {
watch.markTimeEnd("getLastAvail");
}
}
if (debug) {
log.debug(watch);
}
return data;
}
private Map<Integer, MetricValue> getLastDataPts(Collection<Integer> mids, long timestamp) {
final int BATCH_SIZE = 500;
final Map<Integer, MetricValue> rtn = new HashMap<Integer, MetricValue>(mids.size());
if ((mids == null) || mids.isEmpty()) {
return rtn;
}
// all cached values are inserted into rtn
// nodata represents values that are not cached
final Collection<Integer> nodata = getCachedDataPoints(mids, rtn, timestamp);
ArrayList<Integer> ids = null;
final boolean debug = log.isDebugEnabled();
if (nodata.isEmpty()) {
if (debug) {
log.debug("got data from cache");
}
// since we have all the data from cache (nodata is empty), just return it
return rtn;
} else {
ids = new ArrayList<Integer>(nodata);
}
Connection conn = null;
Statement stmt = null;
final StopWatch watch = new StopWatch();
try {
conn = dbUtil.getConnection();
stmt = conn.createStatement();
for (int i=0; i<ids.size(); i+=BATCH_SIZE) {
final int max = Math.min(ids.size(), i+BATCH_SIZE);
// Create sub array
Collection<Integer> subids = ids.subList(i, max);
if (debug) {
watch.markTimeBegin("setDataPoints");
}
setDataPoints(rtn, timestamp, subids, stmt);
if (debug) {
watch.markTimeEnd("setDataPoints");
}
}
} catch (SQLException e) {
throw new SystemException("Cannot get last values", e);
} finally {
DBUtil.closeJDBCObjects(LOG_CTX, conn, stmt, null);
if (debug) {
log.debug(watch);
}
}
List<DataPoint> dataPoints = convertMetricId2MetricValueMapToDataPoints(rtn);
updateMetricDataCache(dataPoints);
return rtn;
}
/**
* Get data points from cache only
*/
@Transactional(readOnly = true)
public Collection<Integer> getCachedDataPoints(Collection<Integer> mids,
Map<Integer, MetricValue> data,
long timestamp) {
ArrayList<Integer> nodata = new ArrayList<Integer>();
for (Integer mid : mids) {
if (mid == null) {
continue;
}
final MetricValue mval = metricDataCache.get(mid, timestamp);
if (mval != null) {
data.put(mid, mval);
} else {
nodata.add(mid);
}
}
return nodata;
}
private void setDataPoints(Map<Integer, MetricValue> data, long timestamp, Collection<Integer> measIds,
Statement stmt) throws SQLException {
ResultSet rs = null;
try {
StringBuilder sqlBuf = getLastDataPointsSQL(timestamp, measIds);
if (log.isTraceEnabled()) {
log.trace("getLastDataPoints(): " + sqlBuf);
}
rs = stmt.executeQuery(sqlBuf.toString());
while (rs.next()) {
Integer mid = new Integer(rs.getInt(1));
if (!data.containsKey(mid)) {
MetricValue mval = getMetricValue(rs);
data.put(mid, mval);
metricDataCache.add(mid, mval); // Add to cache to avoid
// lookup
}
}
} finally {
DBUtil.closeResultSet(LOG_CTX, rs);
}
}
private StringBuilder getLastDataPointsSQL(long timestamp, Collection<Integer> measIds) {
String tables = (timestamp != MeasurementConstants.TIMERANGE_UNLIMITED) ?
MeasurementUnionStatementBuilder.getUnionStatement(timestamp, now(), measIds,
measurementDAO.getHQDialect()) :
MeasurementUnionStatementBuilder.getUnionStatement(getPurgeRaw(), measIds,
measurementDAO.getHQDialect());
StringBuilder sqlBuf = new StringBuilder(
"SELECT measurement_id, value, timestamp" + " FROM " + tables + ", " +
"(SELECT measurement_id AS id, MAX(timestamp) AS maxt" + " FROM " + tables +
" WHERE ").append(MeasTabManagerUtil.getMeasInStmt(measIds, false));
if (timestamp != MeasurementConstants.TIMERANGE_UNLIMITED) {
;
}
sqlBuf.append(" AND timestamp >= ").append(timestamp);
sqlBuf.append(" GROUP BY measurement_id) mt").append(
" WHERE timestamp = maxt AND measurement_id = id");
return sqlBuf;
}
private long now() {
return System.currentTimeMillis();
}
/**
* Convert the MetricId->MetricValue map to a list of DataPoints.
*
* @param metricId2MetricValueMap The map to convert.
* @return The list of DataPoints.
*/
private List<DataPoint> convertMetricId2MetricValueMapToDataPoints(Map<Integer, MetricValue> metricId2MetricValueMap) {
List<DataPoint> dataPoints = new ArrayList<DataPoint>(metricId2MetricValueMap.size());
for (Entry<Integer, MetricValue> entry : metricId2MetricValueMap.entrySet()) {
Integer mid = entry.getKey();
MetricValue mval = entry.getValue();
dataPoints.add(new DataPoint(mid, mval));
}
return dataPoints;
}
/**
* Get a Baseline data.
*
*
*/
@Transactional(readOnly = true)
public double[] getBaselineData(Measurement meas, long begin, long end) {
if (meas.getTemplate().getAlias().equalsIgnoreCase("availability")) {
Integer[] mids = new Integer[1];
Integer id = meas.getId();
mids[0] = id;
return availabilityManager.getAggregateData(mids, begin, end).get(id);
} else {
return getBaselineMeasData(meas, begin, end);
}
}
private double[] getBaselineMeasData(Measurement meas, long begin, long end) {
// Check the begin and end times
checkTimeArguments(begin, end);
Integer id = meas.getId();
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
// The table to query from
String table = getDataTable(begin, end, id.intValue());
try {
conn = dbUtil.getConnection();
StringBuilder sqlBuf = new StringBuilder(
"SELECT MIN(value), AVG(value), MAX(value) FROM ").append(table).append(
" WHERE timestamp BETWEEN ").append(begin).append(" AND ").append(end).append(
" AND measurement_id = ").append(id.intValue());
stmt = conn.createStatement();
rs = stmt.executeQuery(sqlBuf.toString());
rs.next(); // Better have some result
double[] data = new double[MeasurementConstants.IND_MAX + 1];
data[MeasurementConstants.IND_MIN] = rs.getDouble(1);
data[MeasurementConstants.IND_AVG] = rs.getDouble(2);
data[MeasurementConstants.IND_MAX] = rs.getDouble(3);
return data;
} catch (SQLException e) {
throw new MeasurementDataSourceException("Can't get baseline data for: " + id, e);
} finally {
DBUtil.closeJDBCObjects(LOG_CTX, conn, stmt, rs);
}
}
/**
* Fetch a map of aggregate data values keyed by metrics given a start and
* stop time range
*
* @param tids The template id's of the Measurement
* @param iids The instance id's of the Measurement
* @param begin The start of the time range
* @param end The end of the time range
* @param useAggressiveRollup uses a measurement rollup table to fetch the
* data if the time range spans more than one data table's max
* timerange
* @return the Map of data points
*
*/
@Transactional(readOnly = true)
public Map<Integer, double[]> getAggregateDataByMetric(Integer[] tids, Integer[] iids,
long begin, long end,
boolean useAggressiveRollup) {
checkTimeArguments(begin, end);
Map<Integer, double[]> rtn = getAggDataByMetric(tids, iids, begin, end, useAggressiveRollup);
Collection<Measurement> metrics = measurementDAO.findAvailMeasurements(tids, iids);
if (metrics.size() > 0) {
Integer[] mids = new Integer[metrics.size()];
Iterator<Measurement> it = metrics.iterator();
for (int i = 0; i < mids.length; i++) {
Measurement m = it.next();
mids[i] = m.getId();
}
rtn.putAll(availabilityManager.getAggregateData(mids, begin, end));
}
return rtn;
}
private Map<Integer, double[]> getAggDataByMetric(Integer[] tids, Integer[] iids, long begin,
long end, boolean useAggressiveRollup) {
// Check the begin and end times
this.checkTimeArguments(begin, end);
begin = TimingVoodoo.roundDownTime(begin, MINUTE);
end = TimingVoodoo.roundDownTime(end, MINUTE);
// Result set
HashMap<Integer, double[]> resMap = new HashMap<Integer, double[]>();
if ((tids.length == 0) || (iids.length == 0)) {
return resMap;
}
// Get the data points and add to the ArrayList
Connection conn = null;
PreparedStatement stmt = null;
ResultSet rs = null;
StopWatch timer = new StopWatch();
StringBuffer iconj = new StringBuffer(DBUtil
.composeConjunctions("instance_id", iids.length));
DBUtil.replacePlaceHolders(iconj, iids);
StringBuilder tconj = new StringBuilder(DBUtil.composeConjunctions("template_id",
tids.length));
try {
conn = dbUtil.getConnection();
// The table to query from
List<Integer> measids = MeasTabManagerUtil.getMeasIds(conn, tids, iids);
String table = getDataTable(begin, end, measids.toArray(new Integer[0]),
useAggressiveRollup);
// Use the already calculated min, max and average on
// compressed tables.
String minMax;
if (usesMetricUnion(begin, end, useAggressiveRollup)) {
minMax = " MIN(value), AVG(value), MAX(value) ";
} else {
minMax = " MIN(minvalue), AVG(value), MAX(maxvalue) ";
}
final String aggregateSQL = "SELECT id, " +
minMax +
" FROM " +
table +
"," +
TAB_MEAS +
" WHERE timestamp BETWEEN ? AND ? AND measurement_id = id " +
" AND " + iconj + " AND " + tconj + " GROUP BY id";
if (log.isTraceEnabled()) {
log.trace("getAggregateDataByMetric(): " + aggregateSQL);
}
stmt = conn.prepareStatement(aggregateSQL);
int i = 1;
stmt.setLong(i++, begin);
stmt.setLong(i++, end);
i = this.setStatementArguments(stmt, i, tids);
try {
rs = stmt.executeQuery();
while (rs.next()) {
double[] data = new double[IND_MAX + 1];
Integer mid = new Integer(rs.getInt(1));
data[IND_MIN] = rs.getDouble(2);
data[IND_AVG] = rs.getDouble(3);
data[IND_MAX] = rs.getDouble(4);
// Put it into the result map
resMap.put(mid, data);
}
} finally {
DBUtil.closeResultSet(LOG_CTX, rs);
}
if (log.isTraceEnabled()) {
log.trace("getAggregateDataByMetric(): Statement query elapsed " + "time: " +
timer.getElapsed());
}
return resMap;
} catch (SQLException e) {
log.debug("getAggregateDataByMetric()", e);
throw new SystemException(e);
} finally {
DBUtil.closeJDBCObjects(LOG_CTX, conn, stmt, null);
}
}
/**
* Fetch a map of aggregate data values keyed by metrics given a start and
* stop time range
*
* @param measurements The id's of the Measurement
* @param begin The start of the time range
* @param end The end of the time range
* @param useAggressiveRollup uses a measurement rollup table to fetch the
* data if the time range spans more than one data table's max
* timerange
* @return the map of data points
*
*/
@Transactional(readOnly = true)
public Map<Integer, double[]> getAggregateDataByMetric(List<Measurement> measurements,
long begin, long end,
boolean useAggressiveRollup) {
checkTimeArguments(begin, end);
List<Integer> avids = new ArrayList<Integer>();
List<Integer> mids = new ArrayList<Integer>();
for (Measurement meas : measurements) {
MeasurementTemplate t = meas.getTemplate();
if (t.isAvailability()) {
avids.add(meas.getId());
} else {
mids.add(meas.getId());
}
}
Map<Integer, double[]> rtn = getAggDataByMetric(mids.toArray(new Integer[0]),
begin, end, useAggressiveRollup);
rtn.putAll(availabilityManager.getAggregateData(avids.toArray(new Integer[0]),
begin, end));
return rtn;
}
private Map<Integer, double[]> getAggDataByMetric(Integer[] mids, long begin, long end,
boolean useAggressiveRollup) {
// Check the begin and end times
this.checkTimeArguments(begin, end);
begin = TimingVoodoo.roundDownTime(begin, MINUTE);
end = TimingVoodoo.roundDownTime(end, MINUTE);
// The table to query from
String table = getDataTable(begin, end, mids, useAggressiveRollup);
// Result set
HashMap<Integer, double[]> resMap = new HashMap<Integer, double[]>();
if (mids.length == 0) {
return resMap;
}
// Get the data points and add to the ArrayList
Connection conn = null;
PreparedStatement stmt = null;
ResultSet rs = null;
StopWatch timer = new StopWatch();
StringBuffer mconj = new StringBuffer(DBUtil.composeConjunctions("measurement_id",
mids.length));
DBUtil.replacePlaceHolders(mconj, mids);
// Use the already calculated min, max and average on
// compressed tables.
String minMax;
if (usesMetricUnion(begin, end, useAggressiveRollup)) {
minMax = " MIN(value), AVG(value), MAX(value), ";
} else {
minMax = " MIN(minvalue), AVG(value), MAX(maxvalue), ";
}
final String aggregateSQL = "SELECT measurement_id, " + minMax + " count(*) " + " FROM " +
table + " WHERE timestamp BETWEEN ? AND ? AND " + mconj +
" GROUP BY measurement_id";
try {
conn = dbUtil.getConnection();
if (log.isTraceEnabled()) {
log.trace("getAggregateDataByMetric(): " + aggregateSQL);
}
stmt = conn.prepareStatement(aggregateSQL);
int i = 1;
stmt.setLong(i++, begin);
stmt.setLong(i++, end);
try {
rs = stmt.executeQuery();
while (rs.next()) {
double[] data = new double[IND_CFG_COUNT + 1];
Integer mid = new Integer(rs.getInt(1));
data[IND_MIN] = rs.getDouble(2);
data[IND_AVG] = rs.getDouble(3);
data[IND_MAX] = rs.getDouble(4);
data[IND_CFG_COUNT] = rs.getDouble(5);
// Put it into the result map
resMap.put(mid, data);
}
} finally {
DBUtil.closeResultSet(LOG_CTX, rs);
}
if (log.isTraceEnabled()) {
log.trace("getAggregateDataByMetric(): Statement query elapsed " + "time: " +
timer.getElapsed());
}
return resMap;
} catch (SQLException e) {
log.debug("getAggregateDataByMetric()", e);
throw new SystemException(e);
} finally {
DBUtil.closeJDBCObjects(LOG_CTX, conn, stmt, null);
}
}
// TODO remove after HE-54 allows injection
@Transactional(readOnly = true)
public Analyzer getAnalyzer() {
boolean analyze = false;
try {
Properties conf = serverConfigManager.getConfig();
if (conf.containsKey(HQConstants.OOBEnabled)) {
analyze = Boolean.valueOf(conf.getProperty(HQConstants.OOBEnabled)).booleanValue();
}
} catch (Exception e) {
log.debug("Error looking up server configs", e);
} finally {
if (analyze) {
return (Analyzer) ProductProperties
.getPropertyInstance("hyperic.hq.measurement.analyzer");
}
}
return null;
}
}