/*
* RHQ Management Platform
* Copyright 2013, Red Hat Middleware LLC, and individual contributors
* as indicated by the @author tags. See the copyright.txt file in the
* distribution for a full listing of individual contributors.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation version 2 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
package org.rhq.server.metrics.migrator.workers;
import static com.datastax.driver.core.querybuilder.QueryBuilder.ttl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import com.datastax.driver.core.ResultSetFuture;
import com.datastax.driver.core.querybuilder.Batch;
import com.datastax.driver.core.querybuilder.QueryBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hibernate.StatelessSession;
import org.rhq.server.metrics.domain.MetricsTable;
import org.rhq.server.metrics.migrator.DataMigrator;
import org.rhq.server.metrics.migrator.DataMigrator.DataMigratorConfiguration;
import org.rhq.server.metrics.migrator.DataMigrator.Task;
import org.rhq.server.metrics.migrator.datasources.ExistingDataSource;
/**
* @author Stefan Negrea
*
*/
public class RawDataMigrator extends AbstractMigrationWorker implements CallableMigrationWorker {
private final Log log = LogFactory.getLog(RawDataMigrator.class);
private final Queue<String> tablesNotProcessed = new LinkedList<String>(Arrays.asList(getRawDataTables()));
private final MetricsIndexMigrator metricsIndexAccumulator;
private final DataMigratorConfiguration config;
public RawDataMigrator(DataMigratorConfiguration config) {
this.config = config;
this.metricsIndexAccumulator = new MetricsIndexMigrator(MigrationTable.RAW, config);
}
public long estimate() throws Exception {
long recordCount = 0;
for (String table : getRawDataTables()) {
String countQuery = String.format(MigrationQuery.COUNT_RAW.toString(), table);
long tableRecordCount = this.getRowCount(countQuery);
log.debug("Retrieved record count for table " + table + " -- " + tableRecordCount);
recordCount += tableRecordCount;
}
Telemetry telemetry = this.performMigration(Task.Estimate);
long estimatedTimeToMigrate = telemetry.getMigrationTime();
long estimation = (recordCount / MAX_RECORDS_TO_LOAD_FROM_SQL / NUMBER_OF_BATCHES_FOR_ESTIMATION)
* estimatedTimeToMigrate;
estimation += telemetry.getNonMigrationTime();
return estimation;
}
public void migrate() throws Exception {
performMigration(Task.Migrate);
}
private long getRowCount(String countQuery) {
StatelessSession session = getSQLSession(config);
org.hibernate.Query query = session.createSQLQuery(countQuery);
query.setReadOnly(true);
query.setTimeout(DataMigrator.SQL_TIMEOUT);
long count = Long.parseLong(query.uniqueResult().toString());
closeSQLSession(session);
return count;
}
private Telemetry performMigration(Task task) throws Exception {
Telemetry telemetry = new Telemetry();
telemetry.getGeneralTimer().start();
long numberOfBatchesMigrated = 0;
List<Object[]> existingData;
int failureCount;
telemetry.getMigrationTimer().start();
telemetry.getMigrationTimer().suspend();
while (!tablesNotProcessed.isEmpty()) {
String table = tablesNotProcessed.peek();
String selectQuery = String.format(MigrationQuery.SELECT_RAW_DATA.toString(), table);
ExistingDataSource dataSource = getExistingDataSource(selectQuery, task, config);
dataSource.initialize();
log.info("Start migrating raw table: " + table);
telemetry.getMigrationTimer().resume();
int lastMigratedRecord = 0;
while (true) {
existingData = dataSource.getData(lastMigratedRecord, MAX_RECORDS_TO_LOAD_FROM_SQL);
if (existingData == null || existingData.size() == 0) {
break;
}
lastMigratedRecord += existingData.size();
failureCount = 0;
while (failureCount < MAX_NUMBER_OF_FAILURES) {
try {
insertDataToCassandra(existingData);
break;
} catch (Exception e) {
log.error("Failed to insert " + MetricsTable.RAW.toString()
+ " data. Attempting to insert the current batch of data one more time");
log.error(e);
failureCount++;
if (failureCount == MAX_AGGREGATE_BATCH_TO_CASSANDRA) {
throw e;
}
}
}
log.info("- " + table + " - " + lastMigratedRecord + " -");
numberOfBatchesMigrated++;
if (Task.Estimate.equals(task) && numberOfBatchesMigrated >= NUMBER_OF_BATCHES_FOR_ESTIMATION) {
break;
}
}
telemetry.getMigrationTimer().suspend();
if (Task.Migrate.equals(task)) {
log.info("Done migrating raw table" + table + "---------------------");
if (config.isDeleteDataImmediatelyAfterMigration()) {
deleteTableData(table);
}
} else if (numberOfBatchesMigrated >= NUMBER_OF_BATCHES_FOR_ESTIMATION) {
break;
}
dataSource.close();
tablesNotProcessed.poll();
}
telemetry.getMigrationTimer().resume();
metricsIndexAccumulator.drain();
telemetry.getMigrationTimer().suspend();
telemetry.getGeneralTimer().stop();
return telemetry;
}
private void deleteTableData(String table) throws Exception {
String deleteQuery = String.format(MigrationQuery.DELETE_RAW_ENTRY.toString(), table);
int failureCount = 0;
while (failureCount < MAX_NUMBER_OF_FAILURES) {
try {
StatelessSession session = getSQLSession(config);
session.getTransaction().begin();
org.hibernate.Query nativeQuery = session.createSQLQuery(deleteQuery);
nativeQuery.executeUpdate();
session.getTransaction().commit();
closeSQLSession(session);
log.info("- " + table + " - Cleaned -");
} catch (Exception e) {
log.error("Failed to delete " + table + " data. Attempting to delete data one more time...");
failureCount++;
if (failureCount == MAX_NUMBER_OF_FAILURES) {
throw e;
}
}
}
}
private void insertDataToCassandra(List<Object[]> existingData) throws Exception {
List<ResultSetFuture> resultSetFutures = new ArrayList<ResultSetFuture>();
Batch batch = QueryBuilder.batch();
int batchSize = 0;
//only need approximate TTL to speed up processing
//given that each batch is processed within seconds, getting the
//system time once per batch has minimal impact on the record retention
long creationTimeMillis;
long itemTTLSeconds;
long currentTimeMillis = System.currentTimeMillis();
long expectedTTLMillis = MigrationTable.RAW.getTTLinMilliseconds();
for (Object[] rawDataPoint : existingData) {
creationTimeMillis = Long.parseLong(rawDataPoint[MigrationQuery.TIMESTAMP_INDEX].toString());
itemTTLSeconds = (expectedTTLMillis - currentTimeMillis + creationTimeMillis) / 1000l;
if (itemTTLSeconds > 0) {
int scheduleId = Integer.parseInt(rawDataPoint[MigrationQuery.SCHEDULE_INDEX].toString());
Date creationTime = new Date(creationTimeMillis);
batch.add(QueryBuilder.insertInto(MetricsTable.RAW.toString()).value("schedule_id", scheduleId)
.value("time", creationTime)
.value("value", Double.parseDouble(rawDataPoint[MigrationQuery.VALUE_INDEX].toString()))
.using(ttl((int) itemTTLSeconds)));
batchSize++;
metricsIndexAccumulator.add(scheduleId, creationTimeMillis);
}
if (batchSize >= MAX_RAW_BATCH_TO_CASSANDRA) {
resultSetFutures.add(config.getSession().executeAsync(batch));
batch = QueryBuilder.batch();
batchSize = 0;
}
}
if (batchSize != 0) {
resultSetFutures.add(config.getSession().executeAsync(batch));
}
for (ResultSetFuture future : resultSetFutures) {
future.get();
}
}
}