AggregateDao.java

/*
 * Copyright 2015-2017 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.glowroot.central.repo;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;

import javax.annotation.Nullable;

import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.ResultSetFuture;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.utils.UUIDs;
import com.google.common.base.Charsets;
import com.google.common.base.Strings;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.hash.Hashing;
import com.google.common.primitives.Ints;
import com.google.protobuf.AbstractMessage;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import org.immutables.value.Value;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.glowroot.agent.api.Instrumentation;
import org.glowroot.central.util.Messages;
import org.glowroot.central.util.MoreFutures;
import org.glowroot.central.util.Sessions;
import org.glowroot.common.config.ConfigDefaults;
import org.glowroot.common.config.StorageConfig;
import org.glowroot.common.live.ImmutableOverviewAggregate;
import org.glowroot.common.live.ImmutablePercentileAggregate;
import org.glowroot.common.live.ImmutableThroughputAggregate;
import org.glowroot.common.live.ImmutableTransactionQuery;
import org.glowroot.common.live.LiveAggregateRepository.OverallQuery;
import org.glowroot.common.live.LiveAggregateRepository.OverviewAggregate;
import org.glowroot.common.live.LiveAggregateRepository.PercentileAggregate;
import org.glowroot.common.live.LiveAggregateRepository.ThroughputAggregate;
import org.glowroot.common.live.LiveAggregateRepository.TransactionQuery;
import org.glowroot.common.model.LazyHistogram;
import org.glowroot.common.model.LazyHistogram.ScratchBuffer;
import org.glowroot.common.model.MutableProfile;
import org.glowroot.common.model.MutableQuery;
import org.glowroot.common.model.OverallErrorSummaryCollector;
import org.glowroot.common.model.OverallSummaryCollector;
import org.glowroot.common.model.ProfileCollector;
import org.glowroot.common.model.QueryCollector;
import org.glowroot.common.model.ServiceCallCollector;
import org.glowroot.common.model.TransactionErrorSummaryCollector;
import org.glowroot.common.model.TransactionErrorSummaryCollector.ErrorSummarySortOrder;
import org.glowroot.common.model.TransactionSummaryCollector;
import org.glowroot.common.model.TransactionSummaryCollector.SummarySortOrder;
import org.glowroot.common.repo.AggregateRepository;
import org.glowroot.common.repo.ConfigRepository;
import org.glowroot.common.repo.ConfigRepository.RollupConfig;
import org.glowroot.common.repo.MutableAggregate;
import org.glowroot.common.repo.MutableThreadStats;
import org.glowroot.common.repo.MutableTimer;
import org.glowroot.common.repo.Utils;
import org.glowroot.common.repo.util.ThreadStatsCreator;
import org.glowroot.common.util.Clock;
import org.glowroot.common.util.OnlyUsedByTests;
import org.glowroot.common.util.Styles;
import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.AdvancedConfig;
import org.glowroot.wire.api.model.AggregateOuterClass.Aggregate;
import org.glowroot.wire.api.model.AggregateOuterClass.OldAggregatesByType;
import org.glowroot.wire.api.model.AggregateOuterClass.OldTransactionAggregate;
import org.glowroot.wire.api.model.ProfileOuterClass.Profile;

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;

public class AggregateDao implements AggregateRepository {

    private static final Logger logger = LoggerFactory.getLogger(AggregateDao.class);

    private static final String LCS = "compaction = { 'class' : 'LeveledCompactionStrategy' }";

    private static final Table summaryTable = ImmutableTable.builder()
            .partialName("summary")
            .addColumns(ImmutableColumn.of("total_duration_nanos", "double"))
            .addColumns(ImmutableColumn.of("transaction_count", "bigint"))
            .summary(true)
            .fromInclusive(false)
            .build();

    private static final Table errorSummaryTable = ImmutableTable.builder()
            .partialName("error_summary")
            .addColumns(ImmutableColumn.of("error_count", "bigint"))
            .addColumns(ImmutableColumn.of("transaction_count", "bigint"))
            .summary(true)
            .fromInclusive(false)
            .build();

    private static final Table overviewTable = ImmutableTable.builder()
            .partialName("overview")
            .addColumns(ImmutableColumn.of("total_duration_nanos", "double"))
            .addColumns(ImmutableColumn.of("transaction_count", "bigint"))
            .addColumns(ImmutableColumn.of("async_transactions", "boolean"))
            .addColumns(ImmutableColumn.of("main_thread_root_timers", "blob"))
            .addColumns(ImmutableColumn.of("aux_thread_root_timers", "blob"))
            .addColumns(ImmutableColumn.of("async_root_timers", "blob"))
            .addColumns(ImmutableColumn.of("main_thread_total_cpu_nanos", "double")) // nullable
            .addColumns(ImmutableColumn.of("main_thread_total_blocked_nanos", "double")) // nullable
            .addColumns(ImmutableColumn.of("main_thread_total_waited_nanos", "double")) // nullable
            .addColumns(ImmutableColumn.of("main_thread_total_allocated_bytes", "double")) // nullable
            .addColumns(ImmutableColumn.of("aux_thread_total_cpu_nanos", "double")) // nullable
            .addColumns(ImmutableColumn.of("aux_thread_total_blocked_nanos", "double")) // nullable
            .addColumns(ImmutableColumn.of("aux_thread_total_waited_nanos", "double")) // nullable
            .addColumns(ImmutableColumn.of("aux_thread_total_allocated_bytes", "double")) // nullable
            .summary(false)
            .fromInclusive(true)
            .build();

    private static final Table histogramTable = ImmutableTable.builder()
            .partialName("histogram")
            .addColumns(ImmutableColumn.of("total_duration_nanos", "double"))
            .addColumns(ImmutableColumn.of("transaction_count", "bigint"))
            .addColumns(ImmutableColumn.of("duration_nanos_histogram", "blob"))
            .summary(false)
            .fromInclusive(true)
            .build();

    private static final Table throughputTable = ImmutableTable.builder()
            .partialName("throughput")
            .addColumns(ImmutableColumn.of("transaction_count", "bigint"))
            .summary(false)
            .fromInclusive(true)
            .build();

    private static final Table queryTable = ImmutableTable.builder()
            .partialName("query")
            .addColumns(ImmutableColumn.of("query_type", "varchar"))
            .addColumns(ImmutableColumn.of("truncated_query_text", "varchar"))
            // empty when truncated_query_text is really full query text
            // (not null since this column must be used in clustering key)
            .addColumns(ImmutableColumn.of("full_query_text_sha1", "varchar"))
            .addColumns(ImmutableColumn.of("total_duration_nanos", "double"))
            .addColumns(ImmutableColumn.of("execution_count", "bigint"))
            .addColumns(ImmutableColumn.of("total_rows", "bigint"))
            .addClusterKey("query_type")
            .addClusterKey("truncated_query_text")
            .addClusterKey("full_query_text_sha1") // need this for uniqueness
            .summary(false)
            .fromInclusive(false)
            .build();

    private static final Table serviceCallTable = ImmutableTable.builder()
            .partialName("service_call")
            .addColumns(ImmutableColumn.of("service_call_type", "varchar"))
            .addColumns(ImmutableColumn.of("service_call_text", "varchar"))
            .addColumns(ImmutableColumn.of("total_duration_nanos", "double"))
            .addColumns(ImmutableColumn.of("execution_count", "bigint"))
            .addClusterKey("service_call_type")
            .addClusterKey("service_call_text")
            .summary(false)
            .fromInclusive(false)
            .build();

    private static final Table mainThreadProfileTable = ImmutableTable.builder()
            .partialName("main_thread_profile")
            .addColumns(ImmutableColumn.of("main_thread_profile", "blob"))
            .summary(false)
            .fromInclusive(false)
            .build();

    private static final Table auxThreadProfileTable = ImmutableTable.builder()
            .partialName("aux_thread_profile")
            .addColumns(ImmutableColumn.of("aux_thread_profile", "blob"))
            .summary(false)
            .fromInclusive(false)
            .build();

    private final Session session;
    private final AgentDao agentDao;
    private final TransactionTypeDao transactionTypeDao;
    private final FullQueryTextDao fullQueryTextDao;
    private final ConfigRepository configRepository;
    private final Clock clock;

    // list index is rollupLevel
    private final Map<Table, List<PreparedStatement>> insertOverallPS;
    private final Map<Table, List<PreparedStatement>> insertTransactionPS;
    private final Map<Table, List<PreparedStatement>> readOverallPS;
    private final Map<Table, List<PreparedStatement>> readOverallForRollupPS;
    private final Map<Table, PreparedStatement> readOverallForRollupFromChildPS;
    private final Map<Table, List<PreparedStatement>> readTransactionPS;
    private final Map<Table, List<PreparedStatement>> readTransactionForRollupPS;
    private final Map<Table, PreparedStatement> readTransactionForRollupFromChildPS;

    private final List<PreparedStatement> existsMainThreadProfileOverallPS;
    private final List<PreparedStatement> existsMainThreadProfileTransactionPS;
    private final List<PreparedStatement> existsAuxThreadProfileOverallPS;
    private final List<PreparedStatement> existsAuxThreadProfileTransactionPS;

    private final List<PreparedStatement> insertNeedsRollup;
    private final List<PreparedStatement> readNeedsRollup;
    private final List<PreparedStatement> deleteNeedsRollup;

    private final PreparedStatement insertNeedsRollupFromChild;
    private final PreparedStatement readNeedsRollupFromChild;
    private final PreparedStatement deleteNeedsRollupFromChild;

    private final ImmutableList<Table> allTables;

    public AggregateDao(Session session, AgentDao agentDao, TransactionTypeDao transactionTypeDao,
            FullQueryTextDao fullQueryTextDao, ConfigRepository configRepository, Clock clock)
            throws Exception {
        this.session = session;
        this.agentDao = agentDao;
        this.transactionTypeDao = transactionTypeDao;
        this.fullQueryTextDao = fullQueryTextDao;
        this.configRepository = configRepository;
        this.clock = clock;

        int count = configRepository.getRollupConfigs().size();
        List<Integer> rollupExpirationHours =
                configRepository.getStorageConfig().rollupExpirationHours();

        allTables = ImmutableList.of(summaryTable, errorSummaryTable, overviewTable,
                histogramTable, throughputTable, queryTable, serviceCallTable,
                mainThreadProfileTable, auxThreadProfileTable);
        Map<Table, List<PreparedStatement>> insertOverallMap = Maps.newHashMap();
        Map<Table, List<PreparedStatement>> insertTransactionMap = Maps.newHashMap();
        Map<Table, List<PreparedStatement>> readOverallMap = Maps.newHashMap();
        Map<Table, List<PreparedStatement>> readOverallForRollupMap = Maps.newHashMap();
        Map<Table, PreparedStatement> readOverallForRollupFromChildMap = Maps.newHashMap();
        Map<Table, List<PreparedStatement>> readTransactionMap = Maps.newHashMap();
        Map<Table, List<PreparedStatement>> readTransactionForRollupMap = Maps.newHashMap();
        Map<Table, PreparedStatement> readTransactionForRollupFromChildMap = Maps.newHashMap();
        for (Table table : allTables) {
            List<PreparedStatement> insertOverallList = Lists.newArrayList();
            List<PreparedStatement> insertTransactionList = Lists.newArrayList();
            List<PreparedStatement> readOverallList = Lists.newArrayList();
            List<PreparedStatement> readOverallForRollupList = Lists.newArrayList();
            List<PreparedStatement> readTransactionList = Lists.newArrayList();
            List<PreparedStatement> readTransactionForRollupList = Lists.newArrayList();
            for (int i = 0; i < count; i++) {
                int expirationHours = rollupExpirationHours.get(i);
                if (table.summary()) {
                    Sessions.createTableWithTWCS(session, createSummaryTableQuery(table, false, i),
                            expirationHours);
                    Sessions.createTableWithTWCS(session, createSummaryTableQuery(table, true, i),
                            expirationHours);
                    insertOverallList.add(session.prepare(insertSummaryPS(table, false, i)));
                    insertTransactionList.add(session.prepare(insertSummaryPS(table, true, i)));
                    readOverallList.add(session.prepare(readSummaryPS(table, false, i)));
                    readOverallForRollupList
                            .add(session.prepare(readSummaryForRollupPS(table, false, i)));
                    readTransactionList.add(session.prepare(readSummaryPS(table, true, i)));
                    readTransactionForRollupList
                            .add(session.prepare(readSummaryForRollupPS(table, true, i)));
                } else {
                    Sessions.createTableWithTWCS(session, createTableQuery(table, false, i),
                            expirationHours);
                    Sessions.createTableWithTWCS(session, createTableQuery(table, true, i),
                            expirationHours);
                    insertOverallList.add(session.prepare(insertPS(table, false, i)));
                    insertTransactionList.add(session.prepare(insertPS(table, true, i)));
                    readOverallList.add(session.prepare(readPS(table, false, i)));
                    readOverallForRollupList.add(session.prepare(readForRollupPS(table, false, i)));
                    readTransactionList.add(session.prepare(readPS(table, true, i)));
                    readTransactionForRollupList
                            .add(session.prepare(readForRollupPS(table, true, i)));
                }
            }
            insertOverallMap.put(table, ImmutableList.copyOf(insertOverallList));
            insertTransactionMap.put(table, ImmutableList.copyOf(insertTransactionList));
            readOverallMap.put(table, ImmutableList.copyOf(readOverallList));
            readOverallForRollupMap.put(table, ImmutableList.copyOf(readOverallForRollupList));
            if (table.summary()) {
                readOverallForRollupFromChildMap.put(table,
                        session.prepare(readSummaryForRollupFromChildPS(table, false, 0)));
            } else {
                readOverallForRollupFromChildMap.put(table,
                        session.prepare(readForRollupFromChildPS(table, false, 0)));
            }
            readTransactionMap.put(table, ImmutableList.copyOf(readTransactionList));
            readTransactionForRollupMap.put(table,
                    ImmutableList.copyOf(readTransactionForRollupList));
            if (table.summary()) {
                readTransactionForRollupFromChildMap.put(table,
                        session.prepare(readSummaryForRollupFromChildPS(table, true, 0)));
            } else {
                readTransactionForRollupFromChildMap.put(table,
                        session.prepare(readForRollupFromChildPS(table, true, 0)));
            }
        }
        this.insertOverallPS = ImmutableMap.copyOf(insertOverallMap);
        this.insertTransactionPS = ImmutableMap.copyOf(insertTransactionMap);
        this.readOverallPS = ImmutableMap.copyOf(readOverallMap);
        this.readOverallForRollupPS = ImmutableMap.copyOf(readOverallForRollupMap);
        this.readOverallForRollupFromChildPS =
                ImmutableMap.copyOf(readOverallForRollupFromChildMap);
        this.readTransactionPS = ImmutableMap.copyOf(readTransactionMap);
        this.readTransactionForRollupPS = ImmutableMap.copyOf(readTransactionForRollupMap);
        this.readTransactionForRollupFromChildPS =
                ImmutableMap.copyOf(readTransactionForRollupFromChildMap);

        List<PreparedStatement> existsMainThreadProfileOverallPS = Lists.newArrayList();
        List<PreparedStatement> existsMainThreadProfileTransactionPS = Lists.newArrayList();
        List<PreparedStatement> existsAuxThreadProfileOverallPS = Lists.newArrayList();
        List<PreparedStatement> existsAuxThreadProfileTransactionPS = Lists.newArrayList();
        for (int i = 0; i < count; i++) {
            existsMainThreadProfileOverallPS
                    .add(session.prepare(existsPS(mainThreadProfileTable, false, i)));
            existsMainThreadProfileTransactionPS
                    .add(session.prepare(existsPS(mainThreadProfileTable, true, i)));
            existsAuxThreadProfileOverallPS
                    .add(session.prepare(existsPS(auxThreadProfileTable, false, i)));
            existsAuxThreadProfileTransactionPS
                    .add(session.prepare(existsPS(auxThreadProfileTable, true, i)));
        }
        this.existsMainThreadProfileOverallPS = existsMainThreadProfileOverallPS;
        this.existsMainThreadProfileTransactionPS = existsMainThreadProfileTransactionPS;
        this.existsAuxThreadProfileOverallPS = existsAuxThreadProfileOverallPS;
        this.existsAuxThreadProfileTransactionPS = existsAuxThreadProfileTransactionPS;

        // since rollup operations are idempotent, any records resurrected after gc_grace_seconds
        // would just create extra work, but not have any other effect
        //
        // 3 hours is chosen to match default max_hint_window_in_ms since hints are stored
        // with a TTL of gc_grace_seconds
        // (see http://www.uberobert.com/cassandra_gc_grace_disables_hinted_handoff)
        long needsRollupGcGraceSeconds = HOURS.toSeconds(3);

        List<PreparedStatement> insertNeedsRollup = Lists.newArrayList();
        List<PreparedStatement> readNeedsRollup = Lists.newArrayList();
        List<PreparedStatement> deleteNeedsRollup = Lists.newArrayList();
        for (int i = 1; i < count; i++) {
            session.execute("create table if not exists aggregate_needs_rollup_" + i
                    + " (agent_rollup varchar, capture_time timestamp, uniqueness timeuuid,"
                    + " transaction_types set<varchar>, primary key (agent_rollup, capture_time,"
                    + " uniqueness)) with gc_grace_seconds = " + needsRollupGcGraceSeconds + " and "
                    + LCS);
            // TTL is used to prevent non-idempotent rolling up of partially expired aggregates
            // (e.g. "needs rollup" record resurrecting due to small gc_grace_seconds)
            insertNeedsRollup.add(session.prepare("insert into aggregate_needs_rollup_" + i
                    + " (agent_rollup, capture_time, uniqueness, transaction_types) values"
                    + " (?, ?, ?, ?) using TTL ?"));
            readNeedsRollup.add(session.prepare("select capture_time, uniqueness, transaction_types"
                    + " from aggregate_needs_rollup_" + i + " where agent_rollup = ?"));
            deleteNeedsRollup.add(session.prepare("delete from aggregate_needs_rollup_" + i
                    + " where agent_rollup = ? and capture_time = ? and uniqueness = ?"));
        }
        this.insertNeedsRollup = insertNeedsRollup;
        this.readNeedsRollup = readNeedsRollup;
        this.deleteNeedsRollup = deleteNeedsRollup;

        session.execute("create table if not exists aggregate_needs_rollup_from_child"
                + " (agent_rollup varchar, capture_time timestamp, uniqueness timeuuid,"
                + " child_agent_rollup varchar, transaction_types set<varchar>,"
                + " primary key (agent_rollup, capture_time, uniqueness))"
                + " with gc_grace_seconds = " + needsRollupGcGraceSeconds + " and " + LCS);
        // TTL is used to prevent non-idempotent rolling up of partially expired aggregates
        // (e.g. "needs rollup" record resurrecting due to small gc_grace_seconds)
        insertNeedsRollupFromChild = session.prepare("insert into aggregate_needs_rollup_from_child"
                + " (agent_rollup, capture_time, uniqueness, child_agent_rollup, transaction_types)"
                + " values (?, ?, ?, ?, ?) using TTL ?");
        readNeedsRollupFromChild = session.prepare("select capture_time, uniqueness,"
                + " child_agent_rollup, transaction_types from aggregate_needs_rollup_from_child"
                + " where agent_rollup = ?");
        deleteNeedsRollupFromChild = session.prepare("delete from aggregate_needs_rollup_from_child"
                + " where agent_rollup = ? and capture_time = ? and uniqueness = ?");
    }

    public void store(String agentId, long captureTime,
            List<OldAggregatesByType> aggregatesByTypeList,
            List<Aggregate.SharedQueryText> initialSharedQueryTexts) throws Exception {
        if (aggregatesByTypeList.isEmpty()) {
            agentDao.updateLastCaptureTime(agentId, captureTime).get();
            return;
        }
        List<String> agentRollupIds = agentDao.readAgentRollupIds(agentId);
        int adjustedTTL = getAdjustedTTL(getTTLs().get(0), captureTime, clock);
        List<ResultSetFuture> futures = Lists.newArrayList();
        List<Aggregate.SharedQueryText> sharedQueryTexts = Lists.newArrayList();
        for (Aggregate.SharedQueryText sharedQueryText : initialSharedQueryTexts) {
            String fullTextSha1 = sharedQueryText.getFullTextSha1();
            if (fullTextSha1.isEmpty()) {
                String fullText = sharedQueryText.getFullText();
                if (fullText.length() > StorageConfig.AGGREGATE_QUERY_TEXT_TRUNCATE) {
                    fullTextSha1 =
                            Hashing.sha1().hashString(fullText, Charsets.UTF_8).toString();
                    futures.addAll(fullQueryTextDao.store(agentId, fullTextSha1, fullText));
                    sharedQueryTexts.add(Aggregate.SharedQueryText.newBuilder()
                            .setTruncatedText(fullText.substring(0,
                                    StorageConfig.AGGREGATE_QUERY_TEXT_TRUNCATE))
                            .setFullTextSha1(fullTextSha1)
                            .build());
                } else {
                    sharedQueryTexts.add(sharedQueryText);
                }
            } else {
                futures.addAll(fullQueryTextDao.updateTTL(agentId, fullTextSha1));
                sharedQueryTexts.add(sharedQueryText);
            }
        }

        // wait for success before proceeding in order to ensure cannot end up with orphaned
        // fullTextSha1
        MoreFutures.waitForAll(futures);
        futures.clear();

        for (OldAggregatesByType aggregatesByType : aggregatesByTypeList) {
            String transactionType = aggregatesByType.getTransactionType();
            Aggregate overallAggregate = aggregatesByType.getOverallAggregate();
            futures.addAll(storeOverallAggregate(agentId, transactionType, captureTime,
                    overallAggregate, sharedQueryTexts, adjustedTTL));
            for (OldTransactionAggregate transactionAggregate : aggregatesByType
                    .getTransactionAggregateList()) {
                futures.addAll(storeTransactionAggregate(agentId, transactionType,
                        transactionAggregate.getTransactionName(), captureTime,
                        transactionAggregate.getAggregate(), sharedQueryTexts, adjustedTTL));
            }
            // wait for success before proceeding in order to ensure cannot end up with
            // "no overview table records found" during a transactionName rollup, since
            // transactionName rollups are based on finding transactionName in summary table
            MoreFutures.waitForAll(futures);
            futures.clear();
            for (OldTransactionAggregate transactionAggregate : aggregatesByType
                    .getTransactionAggregateList()) {
                futures.addAll(storeTransactionSummary(agentId, transactionType,
                        transactionAggregate.getTransactionName(), captureTime,
                        transactionAggregate.getAggregate(), adjustedTTL));
            }
            futures.addAll(transactionTypeDao.store(agentRollupIds, transactionType));
        }
        futures.add(agentDao.updateLastCaptureTime(agentId, captureTime));
        // wait for success before inserting "needs rollup" records
        MoreFutures.waitForAll(futures);
        futures.clear();

        List<RollupConfig> rollupConfigs = configRepository.getRollupConfigs();
        // TODO report checker framework issue that occurs without this suppression
        @SuppressWarnings("assignment.type.incompatible")
        Set<String> transactionTypes = aggregatesByTypeList.stream()
                .map(OldAggregatesByType::getTransactionType).collect(Collectors.toSet());

        int needsRollupAdjustedTTL = getNeedsRollupAdjustedTTL(adjustedTTL, rollupConfigs);
        if (agentRollupIds.size() > 1) {
            BoundStatement boundStatement = insertNeedsRollupFromChild.bind();
            int i = 0;
            boundStatement.setString(i++, agentRollupIds.get(1));
            boundStatement.setTimestamp(i++, new Date(captureTime));
            boundStatement.setUUID(i++, UUIDs.timeBased());
            boundStatement.setString(i++, agentId);
            boundStatement.setSet(i++, transactionTypes);
            boundStatement.setInt(i++, needsRollupAdjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }
        // insert into aggregate_needs_rollup_1
        long intervalMillis = rollupConfigs.get(1).intervalMillis();
        long rollupCaptureTime = Utils.getRollupCaptureTime(captureTime, intervalMillis);
        BoundStatement boundStatement = insertNeedsRollup.get(0).bind();
        int i = 0;
        boundStatement.setString(i++, agentId);
        boundStatement.setTimestamp(i++, new Date(rollupCaptureTime));
        boundStatement.setUUID(i++, UUIDs.timeBased());
        boundStatement.setSet(i++, transactionTypes);
        boundStatement.setInt(i++, needsRollupAdjustedTTL);
        futures.add(session.executeAsync(boundStatement));
        MoreFutures.waitForAll(futures);
    }

    // query.from() is non-inclusive
    @Override
    public void mergeOverallSummaryInto(String agentRollupId, OverallQuery query,
            OverallSummaryCollector collector) {
        // currently have to do aggregation client-site (don't want to require Cassandra 2.2 yet)
        ResultSet results = createBoundStatement(agentRollupId, query, summaryTable);
        for (Row row : results) {
            int i = 0;
            // results are ordered by capture time so Math.max() is not needed here
            long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
            double totalDurationNanos = row.getDouble(i++);
            long transactionCount = row.getLong(i++);
            collector.mergeSummary(totalDurationNanos, transactionCount, captureTime);
        }
    }

    // sortOrder and limit are only used by embedded H2 repository, while the central cassandra
    // repository which currently has to pull in all records anyways just delegates ordering and
    // limit to TransactionSummaryCollector
    //
    // query.from() is non-inclusive
    @Override
    public void mergeTransactionSummariesInto(String agentRollupId, OverallQuery query,
            SummarySortOrder sortOrder, int limit, TransactionSummaryCollector collector) {
        // currently have to do group by / sort / limit client-side
        BoundStatement boundStatement =
                checkNotNull(readTransactionPS.get(summaryTable)).get(query.rollupLevel()).bind();
        bindQuery(boundStatement, agentRollupId, query);
        ResultSet results = session.execute(boundStatement);
        for (Row row : results) {
            int i = 0;
            long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
            String transactionName = checkNotNull(row.getString(i++));
            double totalDurationNanos = row.getDouble(i++);
            long transactionCount = row.getLong(i++);
            collector.collect(transactionName, totalDurationNanos, transactionCount, captureTime);
        }
    }

    // query.from() is non-inclusive
    @Override
    public void mergeOverallErrorSummaryInto(String agentRollupId, OverallQuery query,
            OverallErrorSummaryCollector collector) {
        // currently have to do aggregation client-site (don't want to require Cassandra 2.2 yet)
        ResultSet results = createBoundStatement(agentRollupId, query, errorSummaryTable);
        for (Row row : results) {
            int i = 0;
            // results are ordered by capture time so Math.max() is not needed here
            long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
            long errorCount = row.getLong(i++);
            long transactionCount = row.getLong(i++);
            collector.mergeErrorSummary(errorCount, transactionCount, captureTime);
        }
    }

    // sortOrder and limit are only used by embedded H2 repository, while the central cassandra
    // repository which currently has to pull in all records anyways just delegates ordering and
    // limit to TransactionErrorSummaryCollector
    //
    // query.from() is non-inclusive
    @Override
    public void mergeTransactionErrorSummariesInto(String agentRollupId, OverallQuery query,
            ErrorSummarySortOrder sortOrder, int limit,
            TransactionErrorSummaryCollector collector) {
        // currently have to do group by / sort / limit client-side
        BoundStatement boundStatement = checkNotNull(readTransactionPS.get(errorSummaryTable))
                .get(query.rollupLevel()).bind();
        bindQuery(boundStatement, agentRollupId, query);
        ResultSet results = session.execute(boundStatement);
        for (Row row : results) {
            int i = 0;
            long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
            String transactionName = checkNotNull(row.getString(i++));
            long errorCount = row.getLong(i++);
            long transactionCount = row.getLong(i++);
            collector.collect(transactionName, errorCount, transactionCount, captureTime);
        }
    }

    // query.from() is INCLUSIVE
    @Override
    public List<OverviewAggregate> readOverviewAggregates(String agentRollupId,
            TransactionQuery query) throws IOException {
        ResultSet results = executeQuery(agentRollupId, query, overviewTable);
        List<OverviewAggregate> overviewAggregates = Lists.newArrayList();
        for (Row row : results) {
            int i = 0;
            long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
            double totalDurationNanos = row.getDouble(i++);
            long transactionCount = row.getLong(i++);
            boolean asyncTransactions = row.getBool(i++);
            List<Aggregate.Timer> mainThreadRootTimers =
                    Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
            List<Aggregate.Timer> auxThreadRootTimers =
                    Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
            List<Aggregate.Timer> asyncTimers =
                    Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
            ImmutableOverviewAggregate.Builder builder = ImmutableOverviewAggregate.builder()
                    .captureTime(captureTime)
                    .totalDurationNanos(totalDurationNanos)
                    .transactionCount(transactionCount)
                    .asyncTransactions(asyncTransactions)
                    .addAllMainThreadRootTimers(mainThreadRootTimers)
                    .addAllAuxThreadRootTimers(auxThreadRootTimers)
                    .addAllAsyncTimers(asyncTimers);
            Double mainThreadTotalCpuNanos = row.get(i++, Double.class);
            Double mainThreadTotalBlockedNanos = row.get(i++, Double.class);
            Double mainThreadTotalWaitedNanos = row.get(i++, Double.class);
            Double mainThreadTotalAllocatedBytes = row.get(i++, Double.class);
            Aggregate.ThreadStats mainThreadStats =
                    ThreadStatsCreator.create(mainThreadTotalCpuNanos, mainThreadTotalBlockedNanos,
                            mainThreadTotalWaitedNanos, mainThreadTotalAllocatedBytes);
            if (mainThreadStats != null) {
                builder.mainThreadStats(mainThreadStats);
            }
            Double auxThreadTotalCpuNanos = row.get(i++, Double.class);
            Double auxThreadTotalBlockedNanos = row.get(i++, Double.class);
            Double auxThreadTotalWaitedNanos = row.get(i++, Double.class);
            Double auxThreadTotalAllocatedBytes = row.get(i++, Double.class);
            Aggregate.ThreadStats auxThreadStats =
                    ThreadStatsCreator.create(auxThreadTotalCpuNanos, auxThreadTotalBlockedNanos,
                            auxThreadTotalWaitedNanos, auxThreadTotalAllocatedBytes);
            if (auxThreadStats != null) {
                builder.auxThreadStats(auxThreadStats);
            }
            overviewAggregates.add(builder.build());
        }
        return overviewAggregates;
    }

    // query.from() is INCLUSIVE
    @Override
    public List<PercentileAggregate> readPercentileAggregates(String agentRollupId,
            TransactionQuery query) throws InvalidProtocolBufferException {
        ResultSet results = executeQuery(agentRollupId, query, histogramTable);
        List<PercentileAggregate> percentileAggregates = Lists.newArrayList();
        for (Row row : results) {
            int i = 0;
            long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
            double totalDurationNanos = row.getDouble(i++);
            long transactionCount = row.getLong(i++);
            ByteBuffer bytes = checkNotNull(row.getBytes(i++));
            Aggregate.Histogram durationNanosHistogram =
                    Aggregate.Histogram.parseFrom(ByteString.copyFrom(bytes));
            percentileAggregates.add(ImmutablePercentileAggregate.builder()
                    .captureTime(captureTime)
                    .totalDurationNanos(totalDurationNanos)
                    .transactionCount(transactionCount)
                    .durationNanosHistogram(durationNanosHistogram)
                    .build());
        }
        return percentileAggregates;
    }

    // query.from() is INCLUSIVE
    @Override
    public List<ThroughputAggregate> readThroughputAggregates(String agentRollupId,
            TransactionQuery query) throws IOException {
        ResultSet results = executeQuery(agentRollupId, query, throughputTable);
        List<ThroughputAggregate> throughputAggregates = Lists.newArrayList();
        for (Row row : results) {
            long captureTime = checkNotNull(row.getTimestamp(0)).getTime();
            long transactionCount = row.getLong(1);
            throughputAggregates.add(ImmutableThroughputAggregate.builder()
                    .captureTime(captureTime)
                    .transactionCount(transactionCount)
                    .build());
        }
        return throughputAggregates;
    }

    @Override
    public @Nullable String readFullQueryText(String agentRollupId, String fullQueryTextSha1)
            throws Exception {
        return fullQueryTextDao.getFullText(agentRollupId, fullQueryTextSha1);
    }

    // query.from() is non-inclusive
    @Override
    public void mergeQueriesInto(String agentRollupId, TransactionQuery query,
            QueryCollector collector) throws IOException {
        ResultSet results = executeQuery(agentRollupId, query, queryTable);
        long captureTime = Long.MIN_VALUE;
        for (Row row : results) {
            int i = 0;
            captureTime = Math.max(captureTime, checkNotNull(row.getTimestamp(i++)).getTime());
            String queryType = checkNotNull(row.getString(i++));
            String truncatedText = checkNotNull(row.getString(i++));
            // full_query_text_sha1 cannot be null since it is used in clustering key
            String fullTextSha1 = Strings.emptyToNull(row.getString(i++));
            double totalDurationNanos = row.getDouble(i++);
            long executionCount = row.getLong(i++);
            boolean hasTotalRows = !row.isNull(i);
            long totalRows = row.getLong(i++);
            collector.mergeQuery(queryType, truncatedText, fullTextSha1, totalDurationNanos,
                    executionCount, hasTotalRows, totalRows);
            collector.updateLastCaptureTime(captureTime);
        }
    }

    // query.from() is non-inclusive
    @Override
    public void mergeServiceCallsInto(String agentRollupId, TransactionQuery query,
            ServiceCallCollector collector) throws IOException {
        ResultSet results = executeQuery(agentRollupId, query, serviceCallTable);
        long captureTime = Long.MIN_VALUE;
        for (Row row : results) {
            int i = 0;
            captureTime = Math.max(captureTime, checkNotNull(row.getTimestamp(i++)).getTime());
            String serviceCallType = checkNotNull(row.getString(i++));
            String serviceCallText = checkNotNull(row.getString(i++));
            double totalDurationNanos = row.getDouble(i++);
            long executionCount = row.getLong(i++);
            collector.mergeServiceCall(serviceCallType, serviceCallText, totalDurationNanos,
                    executionCount);
            collector.updateLastCaptureTime(captureTime);
        }
    }

    // query.from() is non-inclusive
    @Override
    public void mergeMainThreadProfilesInto(String agentRollupId, TransactionQuery query,
            ProfileCollector collector) throws InvalidProtocolBufferException {
        mergeProfilesInto(agentRollupId, query, mainThreadProfileTable, collector);
    }

    // query.from() is non-inclusive
    @Override
    public void mergeAuxThreadProfilesInto(String agentRollupId, TransactionQuery query,
            ProfileCollector collector) throws InvalidProtocolBufferException {
        mergeProfilesInto(agentRollupId, query, auxThreadProfileTable, collector);
    }

    // query.from() is non-inclusive
    @Override
    public boolean hasMainThreadProfile(String agentRollupId, TransactionQuery query)
            throws Exception {
        BoundStatement boundStatement = query.transactionName() == null
                ? existsMainThreadProfileOverallPS.get(query.rollupLevel()).bind()
                : existsMainThreadProfileTransactionPS.get(query.rollupLevel()).bind();
        bindQuery(boundStatement, agentRollupId, query);
        ResultSet results = session.execute(boundStatement);
        return results.one() != null;
    }

    // query.from() is non-inclusive
    @Override
    public boolean hasAuxThreadProfile(String agentRollupId, TransactionQuery query)
            throws Exception {
        BoundStatement boundStatement = query.transactionName() == null
                ? existsAuxThreadProfileOverallPS.get(query.rollupLevel()).bind()
                : existsAuxThreadProfileTransactionPS.get(query.rollupLevel()).bind();
        bindQuery(boundStatement, agentRollupId, query);
        ResultSet results = session.execute(boundStatement);
        return results.one() != null;
    }

    // query.from() is non-inclusive
    @Override
    public boolean shouldHaveQueries(String agentRollupId, TransactionQuery query) {
        // TODO (this is only used for displaying data expired message)
        return false;
    }

    // query.from() is non-inclusive
    @Override
    public boolean shouldHaveServiceCalls(String agentRollupId, TransactionQuery query) {
        // TODO (this is only used for displaying data expired message)
        return false;
    }

    // query.from() is non-inclusive
    @Override
    public boolean shouldHaveMainThreadProfile(String agentRollupId, TransactionQuery query) {
        // TODO (this is only used for displaying data expired message)
        return false;
    }

    // query.from() is non-inclusive
    @Override
    public boolean shouldHaveAuxThreadProfile(String agentRollupId, TransactionQuery query) {
        // TODO (this is only used for displaying data expired message)
        return false;
    }

    @OnlyUsedByTests
    void truncateAll() {
        for (Table table : allTables) {
            for (int i = 0; i < configRepository.getRollupConfigs().size(); i++) {
                session.execute("truncate " + getTableName(table.partialName(), false, i));
                session.execute("truncate " + getTableName(table.partialName(), true, i));
            }
        }
        for (int i = 1; i < configRepository.getRollupConfigs().size(); i++) {
            session.execute("truncate aggregate_needs_rollup_" + i);
        }
        session.execute("truncate aggregate_needs_rollup_from_child");
    }

    @Instrumentation.Transaction(transactionType = "Background",
            transactionName = "Rollup aggregates", traceHeadline = "Rollup aggregates: {{0}}",
            timer = "rollup aggregates")
    public void rollup(String agentRollupId, @Nullable String parentAgentRollupId, boolean leaf)
            throws Exception {
        List<Integer> ttls = getTTLs();
        if (!leaf) {
            rollupFromChildren(agentRollupId, parentAgentRollupId, ttls.get(0));
        }
        int rollupLevel = 1;
        while (rollupLevel < configRepository.getRollupConfigs().size()) {
            int ttl = ttls.get(rollupLevel);
            rollup(agentRollupId, rollupLevel, ttl);
            rollupLevel++;
        }
    }

    private void rollupFromChildren(String agentRollupId, @Nullable String parentAgentRollupId,
            int ttl) throws Exception {
        final int rollupLevel = 0;
        List<NeedsRollupFromChildren> needsRollupFromChildrenList =
                getNeedsRollupFromChildrenList(agentRollupId, readNeedsRollupFromChild, session);
        List<RollupConfig> rollupConfigs = configRepository.getRollupConfigs();
        long nextRollupIntervalMillis = rollupConfigs.get(rollupLevel + 1).intervalMillis();
        for (NeedsRollupFromChildren needsRollupFromChildren : needsRollupFromChildrenList) {
            long captureTime = needsRollupFromChildren.getCaptureTime();
            int adjustedTTL = getAdjustedTTL(ttl, captureTime, clock);
            int needsRollupAdjustedTTL = getNeedsRollupAdjustedTTL(adjustedTTL, rollupConfigs);
            RollupParams rollupParams = getRollupParams(agentRollupId, rollupLevel, adjustedTTL);
            List<ResultSetFuture> futures = Lists.newArrayList();
            for (Entry<String, Collection<String>> entry : needsRollupFromChildren.getKeys().asMap()
                    .entrySet()) {
                String transactionType = entry.getKey();
                Collection<String> childAgentRollups = entry.getValue();
                futures.addAll(rollupOneFromChildren(rollupParams, transactionType,
                        childAgentRollups, captureTime));
            }
            // wait for above async work to ensure rollup complete before proceeding
            MoreFutures.waitForAll(futures);

            if (parentAgentRollupId != null) {
                // insert needs to happen first before call to postRollup(), see method-level
                // comment on postRollup
                BoundStatement boundStatement = insertNeedsRollupFromChild.bind();
                int i = 0;
                boundStatement.setString(i++, parentAgentRollupId);
                boundStatement.setTimestamp(i++, new Date(captureTime));
                boundStatement.setUUID(i++, UUIDs.timeBased());
                boundStatement.setString(i++, agentRollupId);
                boundStatement.setSet(i++, needsRollupFromChildren.getKeys().keySet());
                boundStatement.setInt(i++, needsRollupAdjustedTTL);
                session.execute(boundStatement);
            }
            postRollup(agentRollupId, needsRollupFromChildren.getCaptureTime(),
                    needsRollupFromChildren.getKeys().keySet(),
                    needsRollupFromChildren.getUniquenessKeysForDeletion(),
                    nextRollupIntervalMillis, insertNeedsRollup.get(rollupLevel),
                    deleteNeedsRollupFromChild, needsRollupAdjustedTTL, session);
        }
    }

    private void rollup(String agentRollupId, int rollupLevel, int ttl) throws Exception {
        List<RollupConfig> rollupConfigs = configRepository.getRollupConfigs();
        long rollupIntervalMillis = rollupConfigs.get(rollupLevel).intervalMillis();
        List<NeedsRollup> needsRollupList = getNeedsRollupList(agentRollupId, rollupLevel,
                rollupIntervalMillis, readNeedsRollup, session, clock);
        Long nextRollupIntervalMillis = null;
        if (rollupLevel + 1 < rollupConfigs.size()) {
            nextRollupIntervalMillis = rollupConfigs.get(rollupLevel + 1).intervalMillis();
        }
        for (NeedsRollup needsRollup : needsRollupList) {
            long captureTime = needsRollup.getCaptureTime();
            int adjustedTTL = getAdjustedTTL(ttl, captureTime, clock);
            int needsRollupAdjustedTTL = getNeedsRollupAdjustedTTL(adjustedTTL, rollupConfigs);
            RollupParams rollupParams = getRollupParams(agentRollupId, rollupLevel, adjustedTTL);
            long from = captureTime - rollupIntervalMillis;
            Set<String> transactionTypes = needsRollup.getKeys();
            List<ResultSetFuture> futures = Lists.newArrayList();
            for (String transactionType : transactionTypes) {
                futures.addAll(rollupOne(rollupParams, transactionType, from, captureTime));
            }
            if (futures.isEmpty()) {
                // no rollups occurred, warning already logged inside rollupOne() above
                // this can happen there is an old "needs rollup" record that was created prior to
                // TTL was introduced in 0.9.6, and when the "last needs rollup" record wasn't
                // processed (also prior to 0.9.6), and when the corresponding old data has expired
                AggregateDao.postRollup(agentRollupId, needsRollup.getCaptureTime(),
                        transactionTypes, needsRollup.getUniquenessKeysForDeletion(), null, null,
                        deleteNeedsRollup.get(rollupLevel - 1), -1, session);
                continue;
            }
            // wait for above async work to ensure rollup complete before proceeding
            MoreFutures.waitForAll(futures);

            PreparedStatement insertNeedsRollup = nextRollupIntervalMillis == null ? null
                    : this.insertNeedsRollup.get(rollupLevel);
            PreparedStatement deleteNeedsRollup = this.deleteNeedsRollup.get(rollupLevel - 1);
            postRollup(agentRollupId, needsRollup.getCaptureTime(), transactionTypes,
                    needsRollup.getUniquenessKeysForDeletion(), nextRollupIntervalMillis,
                    insertNeedsRollup, deleteNeedsRollup, needsRollupAdjustedTTL, session);
        }
    }

    private List<ResultSetFuture> rollupOneFromChildren(RollupParams rollup, String transactionType,
            Collection<String> childAgentRollups, long captureTime) throws Exception {

        ImmutableTransactionQuery query = ImmutableTransactionQuery.builder()
                .transactionType(transactionType)
                .from(captureTime)
                .to(captureTime)
                .rollupLevel(rollup.rollupLevel()) // rolling up from same level (which is always 0)
                .build();
        List<ResultSetFuture> futures = Lists.newArrayList();

        futures.addAll(rollupOverallSummaryFromChildren(rollup, query, childAgentRollups));
        futures.addAll(rollupErrorSummaryFromChildren(rollup, query, childAgentRollups));

        List<String> transactionNames = Lists.newArrayList();
        futures.addAll(rollupTransactionSummaryFromChildren(rollup, query, childAgentRollups,
                transactionNames));
        futures.addAll(rollupTransactionErrorSummaryFromChildren(rollup, query, childAgentRollups));

        ScratchBuffer scratchBuffer = new ScratchBuffer();
        futures.addAll(
                rollupOtherPartsFromChildren(rollup, query, childAgentRollups, scratchBuffer));

        for (String transactionName : transactionNames) {
            futures.addAll(rollupOtherPartsFromChildren(rollup,
                    query.withTransactionName(transactionName), childAgentRollups, scratchBuffer));
        }
        return futures;
    }

    private List<ResultSetFuture> rollupOne(RollupParams rollup, String transactionType, long from,
            long to) throws Exception {

        ImmutableTransactionQuery query = ImmutableTransactionQuery.builder()
                .transactionType(transactionType)
                .from(from)
                .to(to)
                .rollupLevel(rollup.rollupLevel() - 1)
                .build();
        List<ResultSetFuture> futures = Lists.newArrayList();

        futures.addAll(rollupOverallSummary(rollup, query));
        futures.addAll(rollupErrorSummary(rollup, query));

        List<String> transactionNames = Lists.newArrayList();
        futures.addAll(rollupTransactionSummary(rollup, query, transactionNames));
        futures.addAll(rollupTransactionErrorSummary(rollup, query));

        ScratchBuffer scratchBuffer = new ScratchBuffer();
        futures.addAll(rollupOtherParts(rollup, query, scratchBuffer));

        for (String transactionName : transactionNames) {
            futures.addAll(rollupOtherParts(rollup, query.withTransactionName(transactionName),
                    scratchBuffer));
        }
        return futures;
    }

    private List<ResultSetFuture> rollupOtherParts(RollupParams rollup, TransactionQuery query,
            ScratchBuffer scratchBuffer) throws Exception {
        List<ResultSetFuture> futures = Lists.newArrayList();
        futures.addAll(rollupOverview(rollup, query));
        futures.addAll(rollupHistogram(rollup, query, scratchBuffer));
        futures.addAll(rollupThroughput(rollup, query));
        futures.addAll(rollupQueries(rollup, query));
        futures.addAll(rollupServiceCalls(rollup, query));
        futures.addAll(rollupThreadProfile(rollup, query, mainThreadProfileTable));
        futures.addAll(rollupThreadProfile(rollup, query, auxThreadProfileTable));
        return futures;
    }

    private List<ResultSetFuture> rollupOtherPartsFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups,
            ScratchBuffer scratchBuffer)
            throws Exception {
        List<ResultSetFuture> futures = Lists.newArrayList();
        futures.addAll(rollupOverviewFromChildren(rollup, query, childAgentRollups));
        futures.addAll(
                rollupHistogramFromChildren(rollup, query, childAgentRollups, scratchBuffer));
        futures.addAll(rollupThroughputFromChildren(rollup, query, childAgentRollups));
        futures.addAll(rollupQueriesFromChildren(rollup, query, childAgentRollups));
        futures.addAll(rollupServiceCallsFromChildren(rollup, query, childAgentRollups));
        futures.addAll(rollupThreadProfileFromChildren(rollup, query, childAgentRollups,
                mainThreadProfileTable));
        futures.addAll(rollupThreadProfileFromChildren(rollup, query, childAgentRollups,
                auxThreadProfileTable));
        return futures;
    }

    private List<ResultSetFuture> rollupOverallSummary(RollupParams rollup,
            TransactionQuery query) {
        ResultSet results = executeQueryForRollup(rollup.agentRollupId(), query, summaryTable);
        if (results.isExhausted()) {
            // this is unexpected since TTL for "needs rollup" records is shorter than TTL for data
            logger.warn("no summary table records found for agentRollupId={}, query={}",
                    rollup.agentRollupId(), query);
            return ImmutableList.of();
        }
        return rollupOverallSummaryFromRows(rollup, query, results);
    }

    private List<ResultSetFuture> rollupOverallSummaryFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups) {
        List<Row> rows = getRowsForRollupFromChildren(query, childAgentRollups, summaryTable);
        if (rows.isEmpty()) {
            // this is unexpected since TTL for "needs rollup" records is shorter than TTL for data
            logger.warn("no summary table records found for agentRollupId={}, query={}",
                    rollup.agentRollupId(), query);
            return ImmutableList.of();
        }
        return rollupOverallSummaryFromRows(rollup, query, rows);
    }

    private List<ResultSetFuture> rollupOverallSummaryFromRows(RollupParams rollup,
            TransactionQuery query, Iterable<Row> rows) {
        double totalDurationNanos = 0;
        long transactionCount = 0;
        for (Row row : rows) {
            totalDurationNanos += row.getDouble(0);
            transactionCount += row.getLong(1);
        }
        BoundStatement boundStatement =
                getInsertOverallPS(summaryTable, rollup.rollupLevel()).bind();
        int i = 0;
        boundStatement.setString(i++, rollup.agentRollupId());
        boundStatement.setString(i++, query.transactionType());
        boundStatement.setTimestamp(i++, new Date(query.to()));
        boundStatement.setDouble(i++, totalDurationNanos);
        boundStatement.setLong(i++, transactionCount);
        boundStatement.setInt(i++, rollup.adjustedTTL());
        return ImmutableList.of(session.executeAsync(boundStatement));
    }

    private List<ResultSetFuture> rollupErrorSummary(RollupParams rollup, TransactionQuery query) {
        ResultSet results = executeQueryForRollup(rollup.agentRollupId(), query, errorSummaryTable);
        if (results.isExhausted()) {
            return ImmutableList.of();
        }
        return rollupErrorSummaryFromRows(rollup, query, results);
    }

    private List<ResultSetFuture> rollupErrorSummaryFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups) {
        List<Row> rows = getRowsForRollupFromChildren(query, childAgentRollups, errorSummaryTable);
        if (rows.isEmpty()) {
            return ImmutableList.of();
        }
        return rollupErrorSummaryFromRows(rollup, query, rows);
    }

    private List<ResultSetFuture> rollupErrorSummaryFromRows(RollupParams rollup,
            TransactionQuery query, Iterable<Row> rows) {
        long errorCount = 0;
        long transactionCount = 0;
        for (Row row : rows) {
            errorCount += row.getLong(0);
            transactionCount += row.getLong(1);
        }
        BoundStatement boundStatement =
                getInsertOverallPS(errorSummaryTable, rollup.rollupLevel()).bind();
        int i = 0;
        boundStatement.setString(i++, rollup.agentRollupId());
        boundStatement.setString(i++, query.transactionType());
        boundStatement.setTimestamp(i++, new Date(query.to()));
        boundStatement.setLong(i++, errorCount);
        boundStatement.setLong(i++, transactionCount);
        boundStatement.setInt(i++, rollup.adjustedTTL());
        return ImmutableList.of(session.executeAsync(boundStatement));
    }

    // transactionNames is passed in empty, and populated by method
    private List<ResultSetFuture> rollupTransactionSummary(RollupParams rollup,
            TransactionQuery query, List<String> transactionNames) {
        BoundStatement boundStatement = checkNotNull(readTransactionForRollupPS.get(summaryTable))
                .get(query.rollupLevel()).bind();
        bindQuery(boundStatement, rollup.agentRollupId(), query);
        ResultSet results = session.execute(boundStatement);
        if (results.isExhausted()) {
            // this is unexpected since TTL for "needs rollup" records is shorter than TTL for data
            logger.warn("no summary table records found for agentRollupId={}, query={}",
                    rollup.agentRollupId(), query);
            return ImmutableList.of();
        }
        return rollupTransactionSummaryFromRows(rollup, query, results, transactionNames);
    }

    // transactionNames is passed in empty, and populated by method
    private List<ResultSetFuture> rollupTransactionSummaryFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups,
            List<String> transactionNames) {
        List<Row> rows =
                getRowsForSummaryRollupFromChildren(query, childAgentRollups, summaryTable);
        if (rows.isEmpty()) {
            // this is unexpected since TTL for "needs rollup" records is shorter than TTL for data
            logger.warn("no summary table records found for agentRollupId={}, query={}",
                    rollup.agentRollupId(), query);
            return ImmutableList.of();
        }
        return rollupTransactionSummaryFromRows(rollup, query, rows, transactionNames);
    }

    // transactionNames is passed in empty, and populated by method
    private List<ResultSetFuture> rollupTransactionSummaryFromRows(RollupParams rollup,
            TransactionQuery query, Iterable<Row> rows, List<String> transactionNames) {
        BoundStatement boundStatement;
        Map<String, MutableSummary> summaries = Maps.newHashMap();
        for (Row row : rows) {
            int i = 0;
            String transactionName = checkNotNull(row.getString(i++));
            MutableSummary summary = summaries.get(transactionName);
            if (summary == null) {
                summary = new MutableSummary();
                summaries.put(transactionName, summary);
            }
            summary.totalDurationNanos += row.getDouble(i++);
            summary.transactionCount += row.getLong(i++);
        }
        List<ResultSetFuture> futures = Lists.newArrayList();
        PreparedStatement preparedStatement =
                getInsertTransactionPS(summaryTable, rollup.rollupLevel());
        for (Entry<String, MutableSummary> entry : summaries.entrySet()) {
            MutableSummary summary = entry.getValue();
            boundStatement = preparedStatement.bind();
            int i = 0;
            boundStatement.setString(i++, rollup.agentRollupId());
            boundStatement.setString(i++, query.transactionType());
            boundStatement.setTimestamp(i++, new Date(query.to()));
            boundStatement.setString(i++, entry.getKey());
            boundStatement.setDouble(i++, summary.totalDurationNanos);
            boundStatement.setLong(i++, summary.transactionCount);
            boundStatement.setInt(i++, rollup.adjustedTTL());
            futures.add(session.executeAsync(boundStatement));
        }
        transactionNames.addAll(summaries.keySet());
        return futures;
    }

    private List<ResultSetFuture> rollupTransactionErrorSummary(RollupParams rollup,
            TransactionQuery query) {
        BoundStatement boundStatement =
                checkNotNull(readTransactionForRollupPS.get(errorSummaryTable))
                        .get(query.rollupLevel()).bind();
        bindQuery(boundStatement, rollup.agentRollupId(), query);
        ResultSet results = session.execute(boundStatement);
        if (results.isExhausted()) {
            return ImmutableList.of();
        }
        return rollupTransactionErrorSummaryFromRows(rollup, query, results);
    }

    // transactionNames is passed in empty, and populated by method
    private List<ResultSetFuture> rollupTransactionErrorSummaryFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups) {
        List<Row> rows =
                getRowsForSummaryRollupFromChildren(query, childAgentRollups, errorSummaryTable);
        if (rows.isEmpty()) {
            return ImmutableList.of();
        }
        return rollupTransactionErrorSummaryFromRows(rollup, query, rows);
    }

    private List<ResultSetFuture> rollupTransactionErrorSummaryFromRows(RollupParams rollup,
            TransactionQuery query, Iterable<Row> rows) {
        BoundStatement boundStatement;
        Map<String, MutableErrorSummary> summaries = Maps.newHashMap();
        for (Row row : rows) {
            int i = 0;
            String transactionName = checkNotNull(row.getString(i++));
            MutableErrorSummary summary = summaries.get(transactionName);
            if (summary == null) {
                summary = new MutableErrorSummary();
                summaries.put(transactionName, summary);
            }
            summary.errorCount += row.getLong(i++);
            summary.transactionCount += row.getLong(i++);
        }
        PreparedStatement preparedStatement =
                getInsertTransactionPS(errorSummaryTable, rollup.rollupLevel());
        List<ResultSetFuture> futures = Lists.newArrayList();
        for (Entry<String, MutableErrorSummary> entry : summaries.entrySet()) {
            MutableErrorSummary summary = entry.getValue();
            boundStatement = preparedStatement.bind();
            int i = 0;
            boundStatement.setString(i++, rollup.agentRollupId());
            boundStatement.setString(i++, query.transactionType());
            boundStatement.setTimestamp(i++, new Date(query.to()));
            boundStatement.setString(i++, entry.getKey());
            boundStatement.setLong(i++, summary.errorCount);
            boundStatement.setLong(i++, summary.transactionCount);
            boundStatement.setInt(i++, rollup.adjustedTTL());
            futures.add(session.executeAsync(boundStatement));
        }
        return futures;
    }

    private List<ResultSetFuture> rollupOverview(RollupParams rollup, TransactionQuery query)
            throws IOException {
        ResultSet results = executeQueryForRollup(rollup.agentRollupId(), query, overviewTable);
        if (results.isExhausted()) {
            // this is unexpected since TTL for "needs rollup" records is shorter than TTL for data
            logger.warn("no overview table records found for agentRollupId={}, query={}",
                    rollup.agentRollupId(), query);
            return ImmutableList.of();
        }
        return rollupOverviewFromRows(rollup, query, results);
    }

    private List<ResultSetFuture> rollupOverviewFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups) throws IOException {
        List<Row> rows = getRowsForRollupFromChildren(query, childAgentRollups, overviewTable);
        if (rows.isEmpty()) {
            // this is unexpected since TTL for "needs rollup" records is shorter than TTL for data
            logger.warn("no overview table records found for agentRollupId={}, query={}",
                    rollup.agentRollupId(), query);
            return ImmutableList.of();
        }
        return rollupOverviewFromRows(rollup, query, rows);
    }

    private List<ResultSetFuture> rollupOverviewFromRows(RollupParams rollup,
            TransactionQuery query, Iterable<Row> rows) throws IOException {
        double totalDurationNanos = 0;
        long transactionCount = 0;
        boolean asyncTransactions = false;
        List<MutableTimer> mainThreadRootTimers = Lists.newArrayList();
        List<MutableTimer> auxThreadRootTimers = Lists.newArrayList();
        List<MutableTimer> asyncTimers = Lists.newArrayList();
        MutableThreadStats mainThreadStats = new MutableThreadStats();
        MutableThreadStats auxThreadStats = new MutableThreadStats();
        for (Row row : rows) {
            int i = 0;
            totalDurationNanos += row.getDouble(i++);
            transactionCount += row.getLong(i++);
            if (row.getBool(i++)) {
                asyncTransactions = true;
            }
            List<Aggregate.Timer> toBeMergedMainThreadRootTimers =
                    Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
            MutableAggregate.mergeRootTimers(toBeMergedMainThreadRootTimers, mainThreadRootTimers);
            List<Aggregate.Timer> toBeMergedAuxThreadRootTimers =
                    Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
            MutableAggregate.mergeRootTimers(toBeMergedAuxThreadRootTimers, auxThreadRootTimers);
            List<Aggregate.Timer> toBeMergedAsyncTimers =
                    Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
            MutableAggregate.mergeRootTimers(toBeMergedAsyncTimers, asyncTimers);
            mainThreadStats.addTotalCpuNanos(row.get(i++, Double.class));
            mainThreadStats.addTotalBlockedNanos(row.get(i++, Double.class));
            mainThreadStats.addTotalWaitedNanos(row.get(i++, Double.class));
            mainThreadStats.addTotalAllocatedBytes(row.get(i++, Double.class));
            auxThreadStats.addTotalCpuNanos(row.get(i++, Double.class));
            auxThreadStats.addTotalBlockedNanos(row.get(i++, Double.class));
            auxThreadStats.addTotalWaitedNanos(row.get(i++, Double.class));
            auxThreadStats.addTotalAllocatedBytes(row.get(i++, Double.class));
        }
        BoundStatement boundStatement;
        if (query.transactionName() == null) {
            boundStatement = getInsertOverallPS(overviewTable, rollup.rollupLevel()).bind();
        } else {
            boundStatement = getInsertTransactionPS(overviewTable, rollup.rollupLevel()).bind();
        }
        int i = 0;
        boundStatement.setString(i++, rollup.agentRollupId());
        boundStatement.setString(i++, query.transactionType());
        if (query.transactionName() != null) {
            boundStatement.setString(i++, query.transactionName());
        }
        boundStatement.setTimestamp(i++, new Date(query.to()));
        boundStatement.setDouble(i++, totalDurationNanos);
        boundStatement.setLong(i++, transactionCount);
        boundStatement.setBool(i++, asyncTransactions);
        boundStatement.setBytes(i++,
                Messages.toByteBuffer(MutableAggregate.toProto(mainThreadRootTimers)));
        boundStatement.setBytes(i++,
                Messages.toByteBuffer(MutableAggregate.toProto(auxThreadRootTimers)));
        boundStatement.setBytes(i++,
                Messages.toByteBuffer(MutableAggregate.toProto(asyncTimers)));
        boundStatement.setDouble(i++, mainThreadStats.getTotalCpuNanos());
        boundStatement.setDouble(i++, mainThreadStats.getTotalBlockedNanos());
        boundStatement.setDouble(i++, mainThreadStats.getTotalWaitedNanos());
        boundStatement.setDouble(i++, mainThreadStats.getTotalAllocatedBytes());
        boundStatement.setDouble(i++, auxThreadStats.getTotalCpuNanos());
        boundStatement.setDouble(i++, auxThreadStats.getTotalBlockedNanos());
        boundStatement.setDouble(i++, auxThreadStats.getTotalWaitedNanos());
        boundStatement.setDouble(i++, auxThreadStats.getTotalAllocatedBytes());
        boundStatement.setInt(i++, rollup.adjustedTTL());
        return ImmutableList.of(session.executeAsync(boundStatement));
    }

    private List<ResultSetFuture> rollupHistogram(RollupParams rollup, TransactionQuery query,
            ScratchBuffer scratchBuffer) throws Exception {
        ResultSet results = executeQueryForRollup(rollup.agentRollupId(), query, histogramTable);
        if (results.isExhausted()) {
            // this is unexpected since TTL for "needs rollup" records is shorter than TTL for data
            logger.warn("no histogram table records found for agentRollupId={}, query={}",
                    rollup.agentRollupId(), query);
            return ImmutableList.of();
        }
        return rollupHistogramFromRows(rollup, query, results, scratchBuffer);
    }

    private List<ResultSetFuture> rollupHistogramFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups,
            ScratchBuffer scratchBuffer)
            throws Exception {
        List<Row> rows = getRowsForRollupFromChildren(query, childAgentRollups, histogramTable);
        if (rows.isEmpty()) {
            // this is unexpected since TTL for "needs rollup" records is shorter than TTL for data
            logger.warn("no histogram table records found for agentRollupId={}, query={}",
                    rollup.agentRollupId(), query);
            return ImmutableList.of();
        }
        return rollupHistogramFromRows(rollup, query, rows, scratchBuffer);
    }

    private List<ResultSetFuture> rollupHistogramFromRows(RollupParams rollup,
            TransactionQuery query, Iterable<Row> rows, ScratchBuffer scratchBuffer)
            throws Exception {
        double totalDurationNanos = 0;
        long transactionCount = 0;
        LazyHistogram durationNanosHistogram = new LazyHistogram();
        for (Row row : rows) {
            int i = 0;
            totalDurationNanos += row.getDouble(i++);
            transactionCount += row.getLong(i++);
            ByteBuffer bytes = checkNotNull(row.getBytes(i++));
            durationNanosHistogram.merge(Aggregate.Histogram.parseFrom(ByteString.copyFrom(bytes)));
        }
        BoundStatement boundStatement;
        if (query.transactionName() == null) {
            boundStatement = getInsertOverallPS(histogramTable, rollup.rollupLevel()).bind();
        } else {
            boundStatement = getInsertTransactionPS(histogramTable, rollup.rollupLevel()).bind();
        }
        int i = 0;
        boundStatement.setString(i++, rollup.agentRollupId());
        boundStatement.setString(i++, query.transactionType());
        if (query.transactionName() != null) {
            boundStatement.setString(i++, query.transactionName());
        }
        boundStatement.setTimestamp(i++, new Date(query.to()));
        boundStatement.setDouble(i++, totalDurationNanos);
        boundStatement.setLong(i++, transactionCount);
        boundStatement.setBytes(i++, toByteBuffer(durationNanosHistogram.toProto(scratchBuffer)));
        boundStatement.setInt(i++, rollup.adjustedTTL());
        return ImmutableList.of(session.executeAsync(boundStatement));
    }

    private List<ResultSetFuture> rollupThroughput(RollupParams rollup, TransactionQuery query) {
        ResultSet results = executeQueryForRollup(rollup.agentRollupId(), query, throughputTable);
        if (results.isExhausted()) {
            // this is unexpected since TTL for "needs rollup" records is shorter than TTL for data
            logger.warn("no throughput table records found for agentRollupId={}, query={}",
                    rollup.agentRollupId(), query);
            return ImmutableList.of();
        }
        return rollupThroughputFromRows(rollup, query, results);
    }

    private List<ResultSetFuture> rollupThroughputFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups) {
        List<Row> rows = getRowsForRollupFromChildren(query, childAgentRollups, throughputTable);
        if (rows.isEmpty()) {
            // this is unexpected since TTL for "needs rollup" records is shorter than TTL for data
            logger.warn("no throughput table records found for agentRollupId={}, query={}",
                    rollup.agentRollupId(), query);
            return ImmutableList.of();
        }
        return rollupThroughputFromRows(rollup, query, rows);
    }

    private List<ResultSetFuture> rollupThroughputFromRows(RollupParams rollup,
            TransactionQuery query, Iterable<Row> rows) {
        long transactionCount = 0;
        for (Row row : rows) {
            transactionCount += row.getLong(0);
        }
        BoundStatement boundStatement;
        if (query.transactionName() == null) {
            boundStatement = getInsertOverallPS(throughputTable, rollup.rollupLevel()).bind();
        } else {
            boundStatement = getInsertTransactionPS(throughputTable, rollup.rollupLevel()).bind();
        }
        int i = 0;
        boundStatement.setString(i++, rollup.agentRollupId());
        boundStatement.setString(i++, query.transactionType());
        if (query.transactionName() != null) {
            boundStatement.setString(i++, query.transactionName());
        }
        boundStatement.setTimestamp(i++, new Date(query.to()));
        boundStatement.setLong(i++, transactionCount);
        boundStatement.setInt(i++, rollup.adjustedTTL());
        return ImmutableList.of(session.executeAsync(boundStatement));
    }

    private List<ResultSetFuture> rollupQueries(RollupParams rollup, TransactionQuery query)
            throws Exception {
        ResultSet results = executeQueryForRollup(rollup.agentRollupId(), query, queryTable);
        if (results.isExhausted()) {
            return ImmutableList.of();
        }
        return rollupQueriesFromRows(rollup, query, results, false);
    }

    private List<ResultSetFuture> rollupQueriesFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups) throws Exception {
        List<Row> rows = getRowsForRollupFromChildren(query, childAgentRollups, queryTable);
        if (rows.isEmpty()) {
            return ImmutableList.of();
        }
        return rollupQueriesFromRows(rollup, query, rows, true);
    }

    private List<ResultSetFuture> rollupQueriesFromRows(RollupParams rollup, TransactionQuery query,
            Iterable<Row> rows, boolean rollupFromChildren) throws Exception {
        QueryCollector collector = new QueryCollector(rollup.maxAggregateQueriesPerType());
        for (Row row : rows) {
            int i = 0;
            String queryType = checkNotNull(row.getString(i++));
            String truncatedText = checkNotNull(row.getString(i++));
            // full_query_text_sha1 cannot be null since it is used in clustering key
            String fullTextSha1 = Strings.emptyToNull(row.getString(i++));
            double totalDurationNanos = row.getDouble(i++);
            long executionCount = row.getLong(i++);
            boolean hasTotalRows = !row.isNull(i);
            long totalRows = row.getLong(i++);
            collector.mergeQuery(queryType, truncatedText, fullTextSha1, totalDurationNanos,
                    executionCount, hasTotalRows, totalRows);
        }
        return insertQueries(collector.getSortedQueries(), rollup.rollupLevel(),
                rollup.agentRollupId(), query.transactionType(), query.transactionName(),
                query.to(), rollup.adjustedTTL(), rollupFromChildren);
    }

    private List<ResultSetFuture> rollupServiceCalls(RollupParams rollup, TransactionQuery query)
            throws IOException {
        ResultSet results = executeQueryForRollup(rollup.agentRollupId(), query, serviceCallTable);
        if (results.isExhausted()) {
            return ImmutableList.of();
        }
        return rollupServiceCallsFromRows(rollup, query, results);
    }

    private List<ResultSetFuture> rollupServiceCallsFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups) {
        List<Row> rows = getRowsForRollupFromChildren(query, childAgentRollups, serviceCallTable);
        if (rows.isEmpty()) {
            return ImmutableList.of();
        }
        return rollupServiceCallsFromRows(rollup, query, rows);
    }

    private List<ResultSetFuture> rollupServiceCallsFromRows(RollupParams rollup,
            TransactionQuery query, Iterable<Row> rows) {
        ServiceCallCollector collector =
                new ServiceCallCollector(rollup.maxAggregateServiceCallsPerType(), 0);
        for (Row row : rows) {
            int i = 0;
            String serviceCallType = checkNotNull(row.getString(i++));
            String serviceCallText = checkNotNull(row.getString(i++));
            double totalDurationNanos = row.getDouble(i++);
            long executionCount = row.getLong(i++);
            collector.mergeServiceCall(serviceCallType, serviceCallText, totalDurationNanos,
                    executionCount);
        }
        return insertServiceCalls(collector.toProto(), rollup.rollupLevel(), rollup.agentRollupId(),
                query.transactionType(), query.transactionName(), query.to(), rollup.adjustedTTL());
    }

    private List<ResultSetFuture> rollupThreadProfile(RollupParams rollup, TransactionQuery query,
            Table table) throws InvalidProtocolBufferException {
        ResultSet results = executeQueryForRollup(rollup.agentRollupId(), query, table);
        if (results.isExhausted()) {
            return ImmutableList.of();
        }
        return rollupThreadProfileFromRows(rollup, query, results, table);
    }

    private List<ResultSetFuture> rollupThreadProfileFromChildren(RollupParams rollup,
            TransactionQuery query, Collection<String> childAgentRollups, Table table)
            throws InvalidProtocolBufferException {
        List<Row> rows = getRowsForRollupFromChildren(query, childAgentRollups, table);
        if (rows.isEmpty()) {
            return ImmutableList.of();
        }
        return rollupThreadProfileFromRows(rollup, query, rows, table);
    }

    private List<ResultSetFuture> rollupThreadProfileFromRows(RollupParams rollup,
            TransactionQuery query, Iterable<Row> rows, Table table)
            throws InvalidProtocolBufferException {
        MutableProfile profile = new MutableProfile();
        for (Row row : rows) {
            ByteBuffer bytes = checkNotNull(row.getBytes(0));
            profile.merge(Profile.parseFrom(ByteString.copyFrom(bytes)));
        }
        BoundStatement boundStatement;
        if (query.transactionName() == null) {
            boundStatement = getInsertOverallPS(table, rollup.rollupLevel()).bind();
        } else {
            boundStatement = getInsertTransactionPS(table, rollup.rollupLevel()).bind();
        }
        int i = 0;
        boundStatement.setString(i++, rollup.agentRollupId());
        boundStatement.setString(i++, query.transactionType());
        if (query.transactionName() != null) {
            boundStatement.setString(i++, query.transactionName());
        }
        boundStatement.setTimestamp(i++, new Date(query.to()));
        boundStatement.setBytes(i++, toByteBuffer(profile.toProto()));
        boundStatement.setInt(i++, rollup.adjustedTTL());
        return ImmutableList.of(session.executeAsync(boundStatement));
    }

    private List<Row> getRowsForSummaryRollupFromChildren(TransactionQuery query,
            Collection<String> childAgentRollups, Table table) {
        List<Row> rows = Lists.newArrayList();
        for (String childAgentRollup : childAgentRollups) {
            BoundStatement boundStatement =
                    checkNotNull(readTransactionForRollupFromChildPS.get(table)).bind();
            bindQueryForRollupFromChild(boundStatement, childAgentRollup, query);
            for (Row row : session.execute(boundStatement)) {
                rows.add(row);
            }
        }
        return rows;
    }

    private List<Row> getRowsForRollupFromChildren(TransactionQuery query,
            Collection<String> childAgentRollups, Table table) {
        List<Row> rows = Lists.newArrayList();
        for (String childAgentRollup : childAgentRollups) {
            rows.addAll(executeQueryForRollupFromChild(childAgentRollup, query, table));
        }
        return rows;
    }

    private List<ResultSetFuture> storeOverallAggregate(String agentRollupId,
            String transactionType, long captureTime, Aggregate aggregate,
            List<Aggregate.SharedQueryText> sharedQueryTexts, int adjustedTTL) throws Exception {

        final int rollupLevel = 0;

        List<ResultSetFuture> futures = Lists.newArrayList();
        BoundStatement boundStatement = getInsertOverallPS(summaryTable, rollupLevel).bind();
        int i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, transactionType);
        boundStatement.setTimestamp(i++, new Date(captureTime));
        boundStatement.setDouble(i++, aggregate.getTotalDurationNanos());
        boundStatement.setLong(i++, aggregate.getTransactionCount());
        boundStatement.setInt(i++, adjustedTTL);
        futures.add(session.executeAsync(boundStatement));

        if (aggregate.getErrorCount() > 0) {
            boundStatement = getInsertOverallPS(errorSummaryTable, rollupLevel).bind();
            i = 0;
            boundStatement.setString(i++, agentRollupId);
            boundStatement.setString(i++, transactionType);
            boundStatement.setTimestamp(i++, new Date(captureTime));
            boundStatement.setLong(i++, aggregate.getErrorCount());
            boundStatement.setLong(i++, aggregate.getTransactionCount());
            boundStatement.setInt(i++, adjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }

        boundStatement = getInsertOverallPS(overviewTable, rollupLevel).bind();
        i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, transactionType);
        boundStatement.setTimestamp(i++, new Date(captureTime));
        bindAggregate(boundStatement, aggregate, i++, adjustedTTL);
        futures.add(session.executeAsync(boundStatement));

        boundStatement = getInsertOverallPS(histogramTable, rollupLevel).bind();
        i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, transactionType);
        boundStatement.setTimestamp(i++, new Date(captureTime));
        boundStatement.setDouble(i++, aggregate.getTotalDurationNanos());
        boundStatement.setLong(i++, aggregate.getTransactionCount());
        boundStatement.setBytes(i++, toByteBuffer(aggregate.getDurationNanosHistogram()));
        boundStatement.setInt(i++, adjustedTTL);
        futures.add(session.executeAsync(boundStatement));

        boundStatement = getInsertOverallPS(throughputTable, rollupLevel).bind();
        i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, transactionType);
        boundStatement.setTimestamp(i++, new Date(captureTime));
        boundStatement.setLong(i++, aggregate.getTransactionCount());
        boundStatement.setInt(i++, adjustedTTL);
        futures.add(session.executeAsync(boundStatement));

        if (aggregate.hasMainThreadProfile()) {
            Profile profile = aggregate.getMainThreadProfile();
            boundStatement = getInsertOverallPS(mainThreadProfileTable, rollupLevel).bind();
            i = 0;
            boundStatement.setString(i++, agentRollupId);
            boundStatement.setString(i++, transactionType);
            boundStatement.setTimestamp(i++, new Date(captureTime));
            boundStatement.setBytes(i++, toByteBuffer(profile));
            boundStatement.setInt(i++, adjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }
        if (aggregate.hasAuxThreadProfile()) {
            Profile profile = aggregate.getAuxThreadProfile();
            boundStatement = getInsertOverallPS(auxThreadProfileTable, rollupLevel).bind();
            i = 0;
            boundStatement.setString(i++, agentRollupId);
            boundStatement.setString(i++, transactionType);
            boundStatement.setTimestamp(i++, new Date(captureTime));
            boundStatement.setBytes(i++, toByteBuffer(profile));
            boundStatement.setInt(i++, adjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }
        futures.addAll(insertQueries(aggregate.getQueriesByTypeList(), sharedQueryTexts,
                rollupLevel, agentRollupId, transactionType, null, captureTime, adjustedTTL));
        futures.addAll(insertServiceCalls(aggregate.getServiceCallsByTypeList(), rollupLevel,
                agentRollupId, transactionType, null, captureTime, adjustedTTL));
        return futures;
    }

    private List<ResultSetFuture> storeTransactionAggregate(String agentRollupId,
            String transactionType, String transactionName, long captureTime, Aggregate aggregate,
            List<Aggregate.SharedQueryText> sharedQueryTexts, int adjustedTTL) throws IOException {

        final int rollupLevel = 0;

        List<ResultSetFuture> futures = Lists.newArrayList();
        BoundStatement boundStatement = getInsertTransactionPS(overviewTable, rollupLevel).bind();
        int i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, transactionType);
        boundStatement.setString(i++, transactionName);
        boundStatement.setTimestamp(i++, new Date(captureTime));
        bindAggregate(boundStatement, aggregate, i++, adjustedTTL);
        futures.add(session.executeAsync(boundStatement));

        boundStatement = getInsertTransactionPS(histogramTable, rollupLevel).bind();
        i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, transactionType);
        boundStatement.setString(i++, transactionName);
        boundStatement.setTimestamp(i++, new Date(captureTime));
        boundStatement.setDouble(i++, aggregate.getTotalDurationNanos());
        boundStatement.setLong(i++, aggregate.getTransactionCount());
        boundStatement.setBytes(i++, toByteBuffer(aggregate.getDurationNanosHistogram()));
        boundStatement.setInt(i++, adjustedTTL);
        futures.add(session.executeAsync(boundStatement));

        boundStatement = getInsertTransactionPS(throughputTable, rollupLevel).bind();
        i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, transactionType);
        boundStatement.setString(i++, transactionName);
        boundStatement.setTimestamp(i++, new Date(captureTime));
        boundStatement.setLong(i++, aggregate.getTransactionCount());
        boundStatement.setInt(i++, adjustedTTL);
        futures.add(session.executeAsync(boundStatement));

        if (aggregate.hasMainThreadProfile()) {
            Profile profile = aggregate.getMainThreadProfile();
            boundStatement = getInsertTransactionPS(mainThreadProfileTable, rollupLevel).bind();
            i = 0;
            boundStatement.setString(i++, agentRollupId);
            boundStatement.setString(i++, transactionType);
            boundStatement.setString(i++, transactionName);
            boundStatement.setTimestamp(i++, new Date(captureTime));
            boundStatement.setBytes(i++, toByteBuffer(profile));
            boundStatement.setInt(i++, adjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }
        if (aggregate.hasAuxThreadProfile()) {
            Profile profile = aggregate.getAuxThreadProfile();
            boundStatement = getInsertTransactionPS(auxThreadProfileTable, rollupLevel).bind();
            i = 0;
            boundStatement.setString(i++, agentRollupId);
            boundStatement.setString(i++, transactionType);
            boundStatement.setString(i++, transactionName);
            boundStatement.setTimestamp(i++, new Date(captureTime));
            boundStatement.setBytes(i++, toByteBuffer(profile));
            boundStatement.setInt(i++, adjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }
        futures.addAll(
                insertQueries(aggregate.getQueriesByTypeList(), sharedQueryTexts, rollupLevel,
                        agentRollupId, transactionType, transactionName, captureTime, adjustedTTL));
        futures.addAll(insertServiceCalls(aggregate.getServiceCallsByTypeList(), rollupLevel,
                agentRollupId, transactionType, transactionName, captureTime, adjustedTTL));
        return futures;
    }

    private List<ResultSetFuture> storeTransactionSummary(String agentRollupId,
            String transactionType, String transactionName, long captureTime, Aggregate aggregate,
            int adjustedTTL) throws IOException {

        final int rollupLevel = 0;

        List<ResultSetFuture> futures = Lists.newArrayList();
        BoundStatement boundStatement = getInsertTransactionPS(summaryTable, rollupLevel).bind();
        int i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, transactionType);
        boundStatement.setTimestamp(i++, new Date(captureTime));
        boundStatement.setString(i++, transactionName);
        boundStatement.setDouble(i++, aggregate.getTotalDurationNanos());
        boundStatement.setLong(i++, aggregate.getTransactionCount());
        boundStatement.setInt(i++, adjustedTTL);
        futures.add(session.executeAsync(boundStatement));

        if (aggregate.getErrorCount() > 0) {
            boundStatement = getInsertTransactionPS(errorSummaryTable, rollupLevel).bind();
            i = 0;
            boundStatement.setString(i++, agentRollupId);
            boundStatement.setString(i++, transactionType);
            boundStatement.setTimestamp(i++, new Date(captureTime));
            boundStatement.setString(i++, transactionName);
            boundStatement.setLong(i++, aggregate.getErrorCount());
            boundStatement.setLong(i++, aggregate.getTransactionCount());
            boundStatement.setInt(i++, adjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }
        return futures;
    }

    private List<ResultSetFuture> insertQueries(List<Aggregate.QueriesByType> queriesByTypeList,
            List<Aggregate.SharedQueryText> sharedQueryTexts, int rollupLevel, String agentRollupId,
            String transactionType, @Nullable String transactionName, long captureTime,
            int adjustedTTL) {
        List<ResultSetFuture> futures = Lists.newArrayList();
        for (Aggregate.QueriesByType queriesByType : queriesByTypeList) {
            for (Aggregate.Query query : queriesByType.getQueryList()) {
                Aggregate.SharedQueryText sharedQueryText =
                        sharedQueryTexts.get(query.getSharedQueryTextIndex());
                BoundStatement boundStatement;
                if (transactionName == null) {
                    boundStatement = getInsertOverallPS(queryTable, rollupLevel).bind();
                } else {
                    boundStatement = getInsertTransactionPS(queryTable, rollupLevel).bind();
                }
                int i = 0;
                boundStatement.setString(i++, agentRollupId);
                boundStatement.setString(i++, transactionType);
                if (transactionName != null) {
                    boundStatement.setString(i++, transactionName);
                }
                boundStatement.setTimestamp(i++, new Date(captureTime));
                boundStatement.setString(i++, queriesByType.getType());
                String fullTextSha1 = sharedQueryText.getFullTextSha1();
                if (fullTextSha1.isEmpty()) {
                    boundStatement.setString(i++, sharedQueryText.getFullText());
                    // full_query_text_sha1 cannot be null since it is used in clustering key
                    boundStatement.setString(i++, "");
                } else {
                    boundStatement.setString(i++, sharedQueryText.getTruncatedText());
                    boundStatement.setString(i++, fullTextSha1);
                }
                boundStatement.setDouble(i++, query.getTotalDurationNanos());
                boundStatement.setLong(i++, query.getExecutionCount());
                if (query.hasTotalRows()) {
                    boundStatement.setLong(i++, query.getTotalRows().getValue());
                } else {
                    boundStatement.setToNull(i++);
                }
                boundStatement.setInt(i++, adjustedTTL);
                futures.add(session.executeAsync(boundStatement));
            }
        }
        return futures;
    }

    private List<ResultSetFuture> insertQueries(Map<String, List<MutableQuery>> map,
            int rollupLevel, String agentRollupId, String transactionType,
            @Nullable String transactionName, long captureTime, int adjustedTTL,
            boolean rollupFromChildren) throws Exception {
        List<ResultSetFuture> futures = Lists.newArrayList();
        for (Entry<String, List<MutableQuery>> entry : map.entrySet()) {
            for (MutableQuery query : entry.getValue()) {
                BoundStatement boundStatement;
                if (transactionName == null) {
                    boundStatement = getInsertOverallPS(queryTable, rollupLevel).bind();
                } else {
                    boundStatement = getInsertTransactionPS(queryTable, rollupLevel).bind();
                }
                String fullTextSha1 = query.getFullTextSha1();
                int i = 0;
                boundStatement.setString(i++, agentRollupId);
                boundStatement.setString(i++, transactionType);
                if (transactionName != null) {
                    boundStatement.setString(i++, transactionName);
                }
                boundStatement.setTimestamp(i++, new Date(captureTime));
                boundStatement.setString(i++, entry.getKey());
                boundStatement.setString(i++, query.getTruncatedText());
                // full_query_text_sha1 cannot be null since it is used in clustering key
                boundStatement.setString(i++, Strings.nullToEmpty(fullTextSha1));
                boundStatement.setDouble(i++, query.getTotalDurationNanos());
                boundStatement.setLong(i++, query.getExecutionCount());
                if (query.hasTotalRows()) {
                    boundStatement.setLong(i++, query.getTotalRows());
                } else {
                    boundStatement.setToNull(i++);
                }
                boundStatement.setInt(i++, adjustedTTL);
                futures.add(session.executeAsync(boundStatement));
                if (rollupFromChildren && fullTextSha1 != null) {
                    futures.addAll(fullQueryTextDao.updateTTL(agentRollupId, fullTextSha1));
                }
            }
        }
        return futures;
    }

    private List<ResultSetFuture> insertServiceCalls(
            List<Aggregate.ServiceCallsByType> serviceCallsByTypeList, int rollupLevel,
            String agentRollupId, String transactionType, @Nullable String transactionName,
            long captureTime, int adjustedTTL) {
        List<ResultSetFuture> futures = Lists.newArrayList();
        for (Aggregate.ServiceCallsByType serviceCallsByType : serviceCallsByTypeList) {
            for (Aggregate.ServiceCall serviceCall : serviceCallsByType.getServiceCallList()) {
                BoundStatement boundStatement;
                if (transactionName == null) {
                    boundStatement = getInsertOverallPS(serviceCallTable, rollupLevel).bind();
                } else {
                    boundStatement = getInsertTransactionPS(serviceCallTable, rollupLevel).bind();
                }
                int i = 0;
                boundStatement.setString(i++, agentRollupId);
                boundStatement.setString(i++, transactionType);
                if (transactionName != null) {
                    boundStatement.setString(i++, transactionName);
                }
                boundStatement.setTimestamp(i++, new Date(captureTime));
                boundStatement.setString(i++, serviceCallsByType.getType());
                boundStatement.setString(i++, serviceCall.getText());
                boundStatement.setDouble(i++, serviceCall.getTotalDurationNanos());
                boundStatement.setLong(i++, serviceCall.getExecutionCount());
                boundStatement.setInt(i++, adjustedTTL);
                futures.add(session.executeAsync(boundStatement));
            }
        }
        return futures;
    }

    private PreparedStatement getInsertOverallPS(Table table, int rollupLevel) {
        return checkNotNull(insertOverallPS.get(table)).get(rollupLevel);
    }

    private PreparedStatement getInsertTransactionPS(Table table, int rollupLevel) {
        return checkNotNull(insertTransactionPS.get(table)).get(rollupLevel);
    }

    private void bindAggregate(BoundStatement boundStatement, Aggregate aggregate, int startIndex,
            int adjustedTTL) throws IOException {
        int i = startIndex;
        boundStatement.setDouble(i++, aggregate.getTotalDurationNanos());
        boundStatement.setLong(i++, aggregate.getTransactionCount());
        boundStatement.setBool(i++, aggregate.getAsyncTransactions());
        List<Aggregate.Timer> mainThreadRootTimers = aggregate.getMainThreadRootTimerList();
        if (!mainThreadRootTimers.isEmpty()) {
            boundStatement.setBytes(i++, Messages.toByteBuffer(mainThreadRootTimers));
        } else {
            boundStatement.setToNull(i++);
        }
        List<Aggregate.Timer> auxThreadRootTimers = aggregate.getAuxThreadRootTimerList();
        if (!auxThreadRootTimers.isEmpty()) {
            boundStatement.setBytes(i++, Messages.toByteBuffer(auxThreadRootTimers));
        } else {
            boundStatement.setToNull(i++);
        }
        List<Aggregate.Timer> asyncTimers = aggregate.getAsyncTimerList();
        if (!asyncTimers.isEmpty()) {
            boundStatement.setBytes(i++, Messages.toByteBuffer(asyncTimers));
        } else {
            boundStatement.setToNull(i++);
        }
        Aggregate.ThreadStats mainThreadStats = aggregate.getMainThreadStats();
        if (mainThreadStats.hasTotalCpuNanos()) {
            boundStatement.setDouble(i++, mainThreadStats.getTotalCpuNanos().getValue());
        } else {
            boundStatement.setToNull(i++);
        }
        if (mainThreadStats.hasTotalBlockedNanos()) {
            boundStatement.setDouble(i++, mainThreadStats.getTotalBlockedNanos().getValue());
        } else {
            boundStatement.setToNull(i++);
        }
        if (mainThreadStats.hasTotalWaitedNanos()) {
            boundStatement.setDouble(i++, mainThreadStats.getTotalWaitedNanos().getValue());
        } else {
            boundStatement.setToNull(i++);
        }
        if (mainThreadStats.hasTotalAllocatedBytes()) {
            boundStatement.setDouble(i++, mainThreadStats.getTotalAllocatedBytes().getValue());
        } else {
            boundStatement.setToNull(i++);
        }
        Aggregate.ThreadStats auxThreadStats = aggregate.getAuxThreadStats();
        if (auxThreadStats.hasTotalCpuNanos()) {
            boundStatement.setDouble(i++, auxThreadStats.getTotalCpuNanos().getValue());
        } else {
            boundStatement.setToNull(i++);
        }
        if (auxThreadStats.hasTotalBlockedNanos()) {
            boundStatement.setDouble(i++, auxThreadStats.getTotalBlockedNanos().getValue());
        } else {
            boundStatement.setToNull(i++);
        }
        if (auxThreadStats.hasTotalWaitedNanos()) {
            boundStatement.setDouble(i++, auxThreadStats.getTotalWaitedNanos().getValue());
        } else {
            boundStatement.setToNull(i++);
        }
        if (auxThreadStats.hasTotalAllocatedBytes()) {
            boundStatement.setDouble(i++, auxThreadStats.getTotalAllocatedBytes().getValue());
        } else {
            boundStatement.setToNull(i++);
        }
        boundStatement.setInt(i++, adjustedTTL);
    }

    private ResultSet createBoundStatement(String agentRollupId, OverallQuery query, Table table) {
        BoundStatement boundStatement =
                checkNotNull(readOverallPS.get(table)).get(query.rollupLevel()).bind();
        bindQuery(boundStatement, agentRollupId, query);
        return session.execute(boundStatement);
    }

    private ResultSet executeQuery(String agentRollupId, TransactionQuery query, Table table) {
        BoundStatement boundStatement;
        if (query.transactionName() == null) {
            boundStatement = checkNotNull(readOverallPS.get(table)).get(query.rollupLevel()).bind();
        } else {
            boundStatement =
                    checkNotNull(readTransactionPS.get(table)).get(query.rollupLevel()).bind();
        }
        bindQuery(boundStatement, agentRollupId, query);
        return session.execute(boundStatement);
    }

    private ResultSet executeQueryForRollup(String agentRollupId, TransactionQuery query,
            Table table) {
        BoundStatement boundStatement;
        if (query.transactionName() == null) {
            boundStatement =
                    checkNotNull(readOverallForRollupPS.get(table)).get(query.rollupLevel()).bind();
        } else {
            boundStatement = checkNotNull(readTransactionForRollupPS.get(table))
                    .get(query.rollupLevel()).bind();
        }
        bindQuery(boundStatement, agentRollupId, query);
        return session.execute(boundStatement);
    }

    private List<Row> executeQueryForRollupFromChild(String childAgentRollup,
            TransactionQuery query, Table table) {
        BoundStatement boundStatement;
        if (query.transactionName() == null) {
            boundStatement = checkNotNull(readOverallForRollupFromChildPS.get(table)).bind();
        } else {
            boundStatement = checkNotNull(readTransactionForRollupFromChildPS.get(table)).bind();
        }
        bindQueryForRollupFromChild(boundStatement, childAgentRollup, query);
        return session.execute(boundStatement).all();
    }

    private void mergeProfilesInto(String agentRollupId, TransactionQuery query, Table profileTable,
            ProfileCollector collector) throws InvalidProtocolBufferException {
        ResultSet results = executeQuery(agentRollupId, query, profileTable);
        long captureTime = Long.MIN_VALUE;
        for (Row row : results) {
            captureTime = Math.max(captureTime, checkNotNull(row.getTimestamp(0)).getTime());
            ByteBuffer bytes = checkNotNull(row.getBytes(1));
            // TODO optimize this byte copying
            Profile profile = Profile.parseFrom(ByteString.copyFrom(bytes));
            collector.mergeProfile(profile);
            collector.updateLastCaptureTime(captureTime);
        }
    }

    private List<Integer> getTTLs() throws Exception {
        List<Integer> ttls = Lists.newArrayList();
        List<Integer> rollupExpirationHours =
                configRepository.getStorageConfig().rollupExpirationHours();
        for (long expirationHours : rollupExpirationHours) {
            ttls.add(Ints.saturatedCast(HOURS.toSeconds(expirationHours)));
        }
        return ttls;
    }

    private RollupParams getRollupParams(String agentRollupId, int rollupLevel, int adjustedTTL)
            throws Exception {
        ImmutableRollupParams.Builder rollupInfo = ImmutableRollupParams.builder()
                .agentRollupId(agentRollupId)
                .rollupLevel(rollupLevel)
                .adjustedTTL(adjustedTTL);
        AdvancedConfig advancedConfig = configRepository.getAdvancedConfig(agentRollupId);
        if (advancedConfig != null && advancedConfig.hasMaxAggregateQueriesPerType()) {
            rollupInfo.maxAggregateQueriesPerType(
                    advancedConfig.getMaxAggregateQueriesPerType().getValue());
        } else {
            rollupInfo.maxAggregateQueriesPerType(ConfigDefaults.MAX_AGGREGATE_QUERIES_PER_TYPE);
        }
        if (advancedConfig != null && advancedConfig.hasMaxAggregateServiceCallsPerType()) {
            rollupInfo.maxAggregateServiceCallsPerType(
                    advancedConfig.getMaxAggregateServiceCallsPerType().getValue());
        } else {
            rollupInfo.maxAggregateServiceCallsPerType(
                    ConfigDefaults.MAX_AGGREGATE_SERVICE_CALLS_PER_TYPE);
        }
        return rollupInfo.build();
    }

    static int getAdjustedTTL(int ttl, long captureTime, Clock clock) {
        if (ttl == 0) {
            return 0;
        }
        int captureTimeAgoSeconds =
                Ints.saturatedCast(MILLISECONDS.toSeconds(clock.currentTimeMillis() - captureTime));
        int adjustedTTL = ttl - captureTimeAgoSeconds;
        // max is a safety guard
        return Math.max(adjustedTTL, 60);
    }

    static int getNeedsRollupAdjustedTTL(int adjustedTTL, List<RollupConfig> rollupConfigs) {
        if (adjustedTTL == 0) {
            return 0;
        }
        long maxRollupInterval = rollupConfigs.get(rollupConfigs.size() - 1).intervalMillis();
        // reduced by an extra 1 hour to make sure that once needs rollup record is retrieved,
        // there is plenty of time to read the all of the data records in the interval before they
        // expire (reading partially expired interval can lead to non-idempotent rollups)
        int needsRollupAdjustedTTL =
                adjustedTTL - Ints.saturatedCast(MILLISECONDS.toSeconds(maxRollupInterval)) - 3600;
        // max is a safety guard
        return Math.max(needsRollupAdjustedTTL, 60);
    }

    static List<NeedsRollup> getNeedsRollupList(String agentRollupId, int rollupLevel,
            long rollupIntervalMillis, List<PreparedStatement> readNeedsRollup, Session session,
            Clock clock) {
        BoundStatement boundStatement = readNeedsRollup.get(rollupLevel - 1).bind();
        boundStatement.setString(0, agentRollupId);
        ResultSet results = session.execute(boundStatement);
        Map<Long, NeedsRollup> needsRollupMap = Maps.newLinkedHashMap();
        for (Row row : results) {
            int i = 0;
            long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
            UUID uniqueness = row.getUUID(i++);
            Set<String> keys = checkNotNull(row.getSet(i++, String.class));
            NeedsRollup needsRollup = needsRollupMap.get(captureTime);
            if (needsRollup == null) {
                needsRollup = new NeedsRollup(captureTime);
                needsRollupMap.put(captureTime, needsRollup);
            }
            needsRollup.keys.addAll(keys);
            needsRollup.uniquenessKeysForDeletion.add(uniqueness);
        }
        if (needsRollupMap.isEmpty()) {
            return ImmutableList.of();
        }
        List<NeedsRollup> needsRollupList = Lists.newArrayList(needsRollupMap.values());
        NeedsRollup lastNeedsRollup = needsRollupList.get(needsRollupList.size() - 1);
        if (lastNeedsRollup.getCaptureTime() > clock.currentTimeMillis() - rollupIntervalMillis) {
            // normally, the last "needs rollup" capture time is in the near future, so don't roll
            // it up since it is likely still being added to
            //
            // this is mostly to avoid rolling up this data twice, but also currently the UI assumes
            // when it finds rolled up data, it doesn't check for non-rolled up data for same
            // interval
            //
            // the above conditional is to force the rollup of the last "needs rollup" if it is more
            // than one rollup interval in the past, otherwise the last "needs rollup" could expire
            // due to TTL prior to it being rolled up
            needsRollupList.remove(needsRollupList.size() - 1);
        }
        return needsRollupList;
    }

    static List<NeedsRollupFromChildren> getNeedsRollupFromChildrenList(String agentRollupId,
            PreparedStatement readNeedsRollupFromChild, Session session) {
        BoundStatement boundStatement = readNeedsRollupFromChild.bind();
        boundStatement.setString(0, agentRollupId);
        ResultSet results = session.execute(boundStatement);
        Map<Long, NeedsRollupFromChildren> needsRollupFromChildrenMap = Maps.newLinkedHashMap();
        for (Row row : results) {
            int i = 0;
            long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
            UUID uniqueness = row.getUUID(i++);
            String childAgentRollup = checkNotNull(row.getString(i++));
            Set<String> keys = checkNotNull(row.getSet(i++, String.class));
            NeedsRollupFromChildren needsRollup = needsRollupFromChildrenMap.get(captureTime);
            if (needsRollup == null) {
                needsRollup = new NeedsRollupFromChildren(captureTime);
                needsRollupFromChildrenMap.put(captureTime, needsRollup);
            }
            for (String key : keys) {
                needsRollup.keys.put(key, childAgentRollup);
            }
            needsRollup.uniquenessKeysForDeletion.add(uniqueness);
        }
        return ImmutableList.copyOf(needsRollupFromChildrenMap.values());
    }

    // it is important that the insert into next needs_rollup happens after present
    // rollup and before deleting present rollup
    // if insert before present rollup then possible for the next rollup to occur before
    // present rollup has completed
    // if insert after deleting present rollup then possible for error to occur in between
    // and insert would never happen
    static void postRollup(String agentRollupId, long captureTime, Set<String> keys,
            Set<UUID> uniquenessKeysForDeletion, @Nullable Long nextRollupIntervalMillis,
            @Nullable PreparedStatement insertNeedsRollup, PreparedStatement deleteNeedsRollup,
            int needsRollupAdjustedTTL, Session session) throws Exception {
        if (nextRollupIntervalMillis != null) {
            checkNotNull(insertNeedsRollup);
            long rollupCaptureTime = Utils.getRollupCaptureTime(captureTime,
                    nextRollupIntervalMillis);
            BoundStatement boundStatement = insertNeedsRollup.bind();
            int i = 0;
            boundStatement.setString(i++, agentRollupId);
            boundStatement.setTimestamp(i++, new Date(rollupCaptureTime));
            boundStatement.setUUID(i++, UUIDs.timeBased());
            boundStatement.setSet(i++, keys);
            boundStatement.setInt(i++, needsRollupAdjustedTTL);
            // intentionally not async, see method-level comment
            session.execute(boundStatement);
        }
        List<ResultSetFuture> futures = Lists.newArrayList();
        for (UUID uniqueness : uniquenessKeysForDeletion) {
            BoundStatement boundStatement = deleteNeedsRollup.bind();
            int i = 0;
            boundStatement.setString(i++, agentRollupId);
            boundStatement.setTimestamp(i++, new Date(captureTime));
            boundStatement.setUUID(i++, uniqueness);
            futures.add(session.executeAsync(boundStatement));
        }
        MoreFutures.waitForAll(futures);
    }

    private static void bindQuery(BoundStatement boundStatement, String agentRollupId,
            OverallQuery query) {
        int i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, query.transactionType());
        boundStatement.setTimestamp(i++, new Date(query.from()));
        boundStatement.setTimestamp(i++, new Date(query.to()));
    }

    private static void bindQuery(BoundStatement boundStatement, String agentRollupId,
            TransactionQuery query) {
        int i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, query.transactionType());
        String transactionName = query.transactionName();
        if (transactionName != null) {
            boundStatement.setString(i++, transactionName);
        }
        boundStatement.setTimestamp(i++, new Date(query.from()));
        boundStatement.setTimestamp(i++, new Date(query.to()));
    }

    private static void bindQueryForRollupFromChild(BoundStatement boundStatement,
            String agentRollupId, TransactionQuery query) {
        int i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, query.transactionType());
        String transactionName = query.transactionName();
        if (transactionName != null) {
            boundStatement.setString(i++, transactionName);
        }
        boundStatement.setTimestamp(i++, new Date(query.to()));
    }

    private static String createTableQuery(Table table, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("create table if not exists ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" (agent_rollup varchar, transaction_type varchar");
        if (transaction) {
            sb.append(", transaction_name varchar");
        }
        sb.append(", capture_time timestamp");
        for (Column column : table.columns()) {
            sb.append(", ");
            sb.append(column.name());
            sb.append(" ");
            sb.append(column.type());
        }
        sb.append(", primary key ((agent_rollup, transaction_type");
        if (transaction) {
            sb.append(", transaction_name");
        }
        sb.append("), capture_time");
        for (String clusterKey : table.clusterKey()) {
            sb.append(", ");
            sb.append(clusterKey);
        }
        sb.append("))");
        return sb.toString();
    }

    private static String insertPS(Table table, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("insert into ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" (agent_rollup, transaction_type");
        if (transaction) {
            sb.append(", transaction_name");
        }
        sb.append(", capture_time");
        for (Column column : table.columns()) {
            sb.append(", ");
            sb.append(column.name());
        }
        sb.append(") values (?, ?, ?");
        if (transaction) {
            sb.append(", ?");
        }
        sb.append(Strings.repeat(", ?", table.columns().size()));
        sb.append(") using TTL ?");
        return sb.toString();
    }

    private static String readPS(Table table, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("select capture_time");
        for (Column column : table.columns()) {
            sb.append(", ");
            sb.append(column.name());
        }
        sb.append(" from ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" where agent_rollup = ? and transaction_type = ?");
        if (transaction) {
            sb.append(" and transaction_name = ?");
        }
        sb.append(" and capture_time >");
        if (table.fromInclusive()) {
            sb.append("=");
        }
        sb.append(" ? and capture_time <= ?");
        return sb.toString();
    }

    private static String readForRollupPS(Table table, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("select ");
        appendColumnNames(sb, table.columns());
        sb.append(" from ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" where agent_rollup = ? and transaction_type = ?");
        if (transaction) {
            sb.append(" and transaction_name = ?");
        }
        sb.append(" and capture_time > ? and capture_time <= ?");
        return sb.toString();
    }

    private static String readForRollupFromChildPS(Table table, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("select ");
        appendColumnNames(sb, table.columns());
        sb.append(" from ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" where agent_rollup = ? and transaction_type = ?");
        if (transaction) {
            sb.append(" and transaction_name = ?");
        }
        sb.append(" and capture_time = ?");
        return sb.toString();
    }

    private static String existsPS(Table table, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("select agent_rollup");
        sb.append(" from ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" where agent_rollup = ? and transaction_type = ?");
        if (transaction) {
            sb.append(" and transaction_name = ?");
        }
        sb.append(" and capture_time >");
        if (table.fromInclusive()) {
            sb.append("=");
        }
        sb.append(" ? and capture_time <= ? limit 1");
        return sb.toString();
    }

    private static String createSummaryTableQuery(Table table, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("create table if not exists ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" (agent_rollup varchar, transaction_type varchar, capture_time timestamp");
        if (transaction) {
            sb.append(", transaction_name varchar");
        }
        for (Column column : table.columns()) {
            sb.append(", ");
            sb.append(column.name());
            sb.append(" ");
            sb.append(column.type());
        }
        sb.append(", primary key ((agent_rollup, transaction_type), capture_time");
        if (transaction) {
            sb.append(", transaction_name");
        }
        sb.append("))");
        return sb.toString();
    }

    private static String insertSummaryPS(Table table, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("insert into ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" (agent_rollup, transaction_type, capture_time");
        if (transaction) {
            sb.append(", transaction_name");
        }
        for (Column column : table.columns()) {
            sb.append(", ");
            sb.append(column.name());
        }
        sb.append(") values (?, ?, ?");
        if (transaction) {
            sb.append(", ?");
        }
        sb.append(Strings.repeat(", ?", table.columns().size()));
        sb.append(") using TTL ?");
        return sb.toString();
    }

    // currently have to do group by / sort / limit client-side, even on overall_summary
    // because sum(double) requires Cassandra 2.2+
    private static String readSummaryPS(Table table, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        // capture_time is needed to keep track of lastCaptureTime for rollup level when merging
        // recent non-rolled up data
        sb.append("select capture_time");
        if (transaction) {
            sb.append(", transaction_name");
        }
        for (Column column : table.columns()) {
            sb.append(", ");
            sb.append(column.name());
        }
        sb.append(" from ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" where agent_rollup = ? and transaction_type = ? and capture_time >");
        if (table.fromInclusive()) {
            sb.append("=");
        }
        sb.append(" ? and capture_time <= ?");
        return sb.toString();
    }

    private static String readSummaryForRollupPS(Table table, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("select ");
        if (transaction) {
            sb.append("transaction_name, ");
        }
        appendColumnNames(sb, table.columns());
        sb.append(" from ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" where agent_rollup = ? and transaction_type = ? and capture_time > ?");
        sb.append(" and capture_time <= ?");
        return sb.toString();
    }

    private static String readSummaryForRollupFromChildPS(Table table, boolean transaction,
            int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("select ");
        if (transaction) {
            sb.append("transaction_name, ");
        }
        appendColumnNames(sb, table.columns());
        sb.append(" from ");
        sb.append(getTableName(table.partialName(), transaction, i));
        sb.append(" where agent_rollup = ? and transaction_type = ? and capture_time = ?");
        return sb.toString();
    }

    private static String getTableName(String partialName, boolean transaction, int i) {
        StringBuilder sb = new StringBuilder();
        sb.append("aggregate_");
        if (transaction) {
            sb.append("tn_");
        } else {
            sb.append("tt_");
        }
        sb.append(partialName);
        sb.append("_rollup_");
        sb.append(i);
        return sb.toString();
    }

    private static void appendColumnNames(StringBuilder sb, List<Column> columns) {
        boolean addSeparator = false;
        for (Column column : columns) {
            if (addSeparator) {
                sb.append(", ");
            }
            sb.append(column.name());
            addSeparator = true;
        }
    }

    private static ByteBuffer toByteBuffer(AbstractMessage message) {
        return ByteBuffer.wrap(message.toByteString().toByteArray());
    }

    @Value.Immutable
    interface Table {
        String partialName();
        List<Column> columns();
        List<String> clusterKey();
        boolean summary();
        boolean fromInclusive();
    }

    @Value.Immutable
    @Styles.AllParameters
    interface Column {
        String name();
        String type();
    }

    @Value.Immutable
    interface RollupParams {
        String agentRollupId();
        int rollupLevel();
        int adjustedTTL();
        int maxAggregateQueriesPerType();
        int maxAggregateServiceCallsPerType();
    }

    static class NeedsRollup {

        private final long captureTime;
        private final Set<String> keys = Sets.newHashSet(); // transaction types or gauge names
        private final Set<UUID> uniquenessKeysForDeletion = Sets.newHashSet();

        private NeedsRollup(long captureTime) {
            this.captureTime = captureTime;
        }

        long getCaptureTime() {
            return captureTime;
        }

        Set<String> getKeys() {
            return keys;
        }

        Set<UUID> getUniquenessKeysForDeletion() {
            return uniquenessKeysForDeletion;
        }
    }

    static class NeedsRollupFromChildren {

        private final long captureTime;
        // map keys are transaction types or gauge names
        // map values are childAgentRollups
        private final Multimap<String, String> keys = HashMultimap.create();
        private final Set<UUID> uniquenessKeysForDeletion = Sets.newHashSet();

        private NeedsRollupFromChildren(long captureTime) {
            this.captureTime = captureTime;
        }

        long getCaptureTime() {
            return captureTime;
        }

        Multimap<String, String> getKeys() {
            return keys;
        }

        Set<UUID> getUniquenessKeysForDeletion() {
            return uniquenessKeysForDeletion;
        }
    }

    private static class MutableSummary {
        private double totalDurationNanos;
        private long transactionCount;
    }

    private static class MutableErrorSummary {
        private long errorCount;
        private long transactionCount;
    }
}