TraceDao.java

/*
 * Copyright 2015-2017 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.glowroot.central.repo;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.ListIterator;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import javax.annotation.Nullable;

import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.ResultSetFuture;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
import com.google.common.base.Charsets;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.hash.Hashing;
import com.google.common.primitives.Ints;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import org.immutables.value.Value;

import org.glowroot.central.util.Messages;
import org.glowroot.central.util.MoreFutures;
import org.glowroot.central.util.Sessions;
import org.glowroot.common.config.StorageConfig;
import org.glowroot.common.live.ImmutableEntries;
import org.glowroot.common.live.ImmutableTracePoint;
import org.glowroot.common.live.LiveTraceRepository.Entries;
import org.glowroot.common.live.LiveTraceRepository.Existence;
import org.glowroot.common.live.LiveTraceRepository.TracePoint;
import org.glowroot.common.live.LiveTraceRepository.TracePointFilter;
import org.glowroot.common.model.Result;
import org.glowroot.common.repo.ConfigRepository;
import org.glowroot.common.repo.ImmutableErrorMessageCount;
import org.glowroot.common.repo.ImmutableErrorMessagePoint;
import org.glowroot.common.repo.ImmutableErrorMessageResult;
import org.glowroot.common.repo.ImmutableHeaderPlus;
import org.glowroot.common.repo.TraceRepository;
import org.glowroot.common.repo.Utils;
import org.glowroot.common.util.Clock;
import org.glowroot.common.util.Styles;
import org.glowroot.wire.api.model.ProfileOuterClass.Profile;
import org.glowroot.wire.api.model.Proto;
import org.glowroot.wire.api.model.Proto.StackTraceElement;
import org.glowroot.wire.api.model.TraceOuterClass.Trace;

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.concurrent.TimeUnit.HOURS;

public class TraceDao implements TraceRepository {

    private final Session session;
    private final AgentDao agentDao;
    private final TransactionTypeDao transactionTypeDao;
    private final FullQueryTextDao fullQueryTextDao;
    private final TraceAttributeNameDao traceAttributeNameDao;
    private final ConfigRepository configRepository;
    private final Clock clock;

    private final PreparedStatement insertCheck;

    private final PreparedStatement insertOverallSlowPoint;
    private final PreparedStatement insertTransactionSlowPoint;

    private final PreparedStatement insertOverallSlowCount;
    private final PreparedStatement insertTransactionSlowCount;

    private final PreparedStatement insertOverallErrorPoint;
    private final PreparedStatement insertTransactionErrorPoint;

    private final PreparedStatement insertOverallErrorCount;
    private final PreparedStatement insertTransactionErrorCount;

    private final PreparedStatement insertOverallErrorMessage;
    private final PreparedStatement insertTransactionErrorMessage;

    private final PreparedStatement insertHeader;
    private final PreparedStatement insertEntry;
    private final PreparedStatement insertSharedQueryText;
    private final PreparedStatement insertMainThreadProfile;
    private final PreparedStatement insertAuxThreadProfile;

    private final PreparedStatement readCheck;

    private final PreparedStatement readOverallSlowPoint;
    private final PreparedStatement readTransactionSlowPoint;
    private final PreparedStatement readOverallErrorPoint;
    private final PreparedStatement readTransactionErrorPoint;

    private final PreparedStatement readOverallErrorMessage;
    private final PreparedStatement readTransactionErrorMessage;

    private final PreparedStatement readHeader;
    private final PreparedStatement readEntries;
    private final PreparedStatement readSharedQueryTexts;
    private final PreparedStatement readMainThreadProfile;
    private final PreparedStatement readAuxThreadProfile;

    private final PreparedStatement deletePartialOverallSlowPoint;
    private final PreparedStatement deletePartialTransactionSlowPoint;

    private final PreparedStatement deletePartialOverallSlowCount;
    private final PreparedStatement deletePartialTransactionSlowCount;

    public TraceDao(Session session, AgentDao agentDao, TransactionTypeDao transactionTypeDao,
            FullQueryTextDao fullQueryTextDao, TraceAttributeNameDao traceAttributeNameDao,
            ConfigRepository configRepository, Clock clock) throws Exception {
        this.session = session;
        this.agentDao = agentDao;
        this.transactionTypeDao = transactionTypeDao;
        this.fullQueryTextDao = fullQueryTextDao;
        this.traceAttributeNameDao = traceAttributeNameDao;
        this.configRepository = configRepository;
        this.clock = clock;

        int expirationHours = configRepository.getStorageConfig().traceExpirationHours();

        Sessions.createTableWithTWCS(session, "create table if not exists trace_check"
                + " (agent_rollup varchar, agent_id varchar, trace_id varchar, primary key"
                + " ((agent_rollup, agent_id), trace_id))", expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_tt_slow_point"
                + " (agent_rollup varchar, transaction_type varchar, capture_time timestamp,"
                + " agent_id varchar, trace_id varchar, duration_nanos bigint, partial boolean,"
                + " error boolean, headline varchar, user varchar, attributes blob, primary key"
                + " ((agent_rollup, transaction_type), capture_time, agent_id, trace_id))",
                expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_tn_slow_point"
                + " (agent_rollup varchar, transaction_type varchar, transaction_name varchar,"
                + " capture_time timestamp, agent_id varchar, trace_id varchar,"
                + " duration_nanos bigint, partial boolean, error boolean, headline varchar,"
                + " user varchar, attributes blob, primary key ((agent_rollup, transaction_type,"
                + " transaction_name), capture_time, agent_id, trace_id))", expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_tt_error_point"
                + " (agent_rollup varchar, transaction_type varchar, capture_time timestamp,"
                + " agent_id varchar, trace_id varchar, duration_nanos bigint, partial boolean,"
                + " error_message varchar, headline varchar, user varchar, attributes blob,"
                + " primary key ((agent_rollup, transaction_type), capture_time, agent_id,"
                + " trace_id))", expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_tn_error_point"
                + " (agent_rollup varchar, transaction_type varchar, transaction_name varchar,"
                + " capture_time timestamp, agent_id varchar, trace_id varchar,"
                + " duration_nanos bigint, partial boolean, error_message varchar,"
                + " headline varchar, user varchar, attributes blob, primary key ((agent_rollup,"
                + " transaction_type, transaction_name), capture_time, agent_id, trace_id))",
                expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_tt_error_message"
                + " (agent_rollup varchar, transaction_type varchar, capture_time timestamp,"
                + " agent_id varchar, trace_id varchar, error_message varchar, primary key"
                + " ((agent_rollup, transaction_type), capture_time, agent_id, trace_id))",
                expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_tn_error_message"
                + " (agent_rollup varchar, transaction_type varchar, transaction_name varchar,"
                + " capture_time timestamp, agent_id varchar, trace_id varchar,"
                + " error_message varchar, primary key ((agent_rollup, transaction_type,"
                + " transaction_name), capture_time, agent_id, trace_id))", expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_header"
                + " (agent_id varchar, trace_id varchar, header blob, primary key (agent_id,"
                + " trace_id))", expirationHours);

        // index_ is just to provide uniqueness
        Sessions.createTableWithTWCS(session, "create table if not exists trace_entry"
                + " (agent_id varchar, trace_id varchar, index_ int, depth int,"
                + " start_offset_nanos bigint, duration_nanos bigint, active boolean,"
                + " message varchar, shared_query_text_index int, query_message_prefix varchar,"
                + " query_message_suffix varchar, detail blob, location_stack_trace blob,"
                + " error blob, primary key (agent_id, trace_id, index_))", expirationHours);

        // index_ is just to provide uniqueness
        Sessions.createTableWithTWCS(session, "create table if not exists trace_shared_query_text"
                + " (agent_id varchar, trace_id varchar, index_ int, truncated_text varchar,"
                + " truncated_end_text varchar, full_text_sha1 varchar, primary key (agent_id,"
                + " trace_id, index_))", expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_main_thread_profile"
                + " (agent_id varchar, trace_id varchar, profile blob, primary key (agent_id,"
                + " trace_id))", expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_aux_thread_profile"
                + " (agent_id varchar, trace_id varchar, profile blob, primary key (agent_id,"
                + " trace_id))", expirationHours);

        // agent_rollup/capture_time is not necessarily unique
        // using a counter would be nice since only need sum over capture_time range
        // but counter has no TTL, see https://issues.apache.org/jira/browse/CASSANDRA-2103
        // so adding trace_id to provide uniqueness
        Sessions.createTableWithTWCS(session, "create table if not exists trace_tt_slow_count"
                + " (agent_rollup varchar, transaction_type varchar, capture_time timestamp,"
                + " agent_id varchar, trace_id varchar, primary key ((agent_rollup,"
                + " transaction_type), capture_time, agent_id, trace_id))", expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_tn_slow_count"
                + " (agent_rollup varchar, transaction_type varchar, transaction_name varchar,"
                + " capture_time timestamp, agent_id varchar, trace_id varchar, primary key"
                + " ((agent_rollup, transaction_type, transaction_name), capture_time, agent_id,"
                + " trace_id))", expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_tt_error_count"
                + " (agent_rollup varchar, transaction_type varchar, capture_time timestamp,"
                + " agent_id varchar, trace_id varchar, primary key ((agent_rollup,"
                + " transaction_type), capture_time, agent_id, trace_id))", expirationHours);

        Sessions.createTableWithTWCS(session, "create table if not exists trace_tn_error_count"
                + " (agent_rollup varchar, transaction_type varchar, transaction_name varchar,"
                + " capture_time timestamp, agent_id varchar, trace_id varchar, primary key"
                + " ((agent_rollup, transaction_type, transaction_name), capture_time, agent_id,"
                + " trace_id))", expirationHours);

        insertCheck = session.prepare("insert into trace_check (agent_rollup, agent_id, trace_id)"
                + " values (?, ?, ?) using ttl ?");

        insertOverallSlowPoint = session.prepare("insert into trace_tt_slow_point (agent_rollup,"
                + " transaction_type, capture_time, agent_id, trace_id, duration_nanos, partial,"
                + " error, headline, user, attributes) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
                + " using ttl ?");

        insertTransactionSlowPoint = session.prepare("insert into trace_tn_slow_point"
                + " (agent_rollup, transaction_type, transaction_name, capture_time, agent_id,"
                + " trace_id, duration_nanos, partial, error, headline, user, attributes) values"
                + " (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) using ttl ?");

        insertOverallSlowCount = session.prepare("insert into trace_tt_slow_count (agent_rollup,"
                + " transaction_type, capture_time, agent_id, trace_id) values (?, ?, ?, ?, ?)"
                + " using ttl ?");

        insertTransactionSlowCount = session.prepare("insert into trace_tn_slow_count"
                + " (agent_rollup, transaction_type, transaction_name, capture_time, agent_id,"
                + " trace_id) values (?, ?, ?, ?, ?, ?) using ttl ?");

        insertOverallErrorPoint = session.prepare("insert into trace_tt_error_point (agent_rollup,"
                + " transaction_type, capture_time, agent_id, trace_id, duration_nanos, partial,"
                + " error_message, headline, user, attributes) values"
                + " (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) using ttl ?");

        insertTransactionErrorPoint = session.prepare("insert into trace_tn_error_point"
                + " (agent_rollup, transaction_type, transaction_name, capture_time, agent_id,"
                + " trace_id, duration_nanos, partial, error_message, headline, user, attributes)"
                + " values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) using ttl ?");

        insertOverallErrorCount = session.prepare("insert into trace_tt_error_count (agent_rollup,"
                + " transaction_type, capture_time, agent_id, trace_id) values (?, ?, ?, ?, ?)"
                + " using ttl ?");

        insertTransactionErrorCount = session.prepare("insert into trace_tn_error_count"
                + " (agent_rollup, transaction_type, transaction_name, capture_time, agent_id,"
                + " trace_id) values (?, ?, ?, ?, ?, ?) using ttl ?");

        insertOverallErrorMessage = session.prepare("insert into trace_tt_error_message"
                + " (agent_rollup, transaction_type, capture_time, agent_id, trace_id,"
                + " error_message) values (?, ?, ?, ?, ?, ?) using ttl ?");

        insertTransactionErrorMessage = session.prepare("insert into trace_tn_error_message"
                + " (agent_rollup, transaction_type, transaction_name, capture_time, agent_id,"
                + " trace_id, error_message) values (?, ?, ?, ?, ?, ?, ?) using ttl ?");

        insertHeader = session.prepare("insert into trace_header (agent_id, trace_id, header)"
                + " values (?, ?, ?) using ttl ?");

        insertEntry = session.prepare("insert into trace_entry (agent_id, trace_id, index_, depth,"
                + " start_offset_nanos, duration_nanos, active, message, shared_query_text_index,"
                + " query_message_prefix, query_message_suffix, detail, location_stack_trace,"
                + " error) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) using ttl ?");

        insertSharedQueryText = session.prepare("insert into trace_shared_query_text (agent_id,"
                + " trace_id, index_, truncated_text, truncated_end_text, full_text_sha1) values"
                + " (?, ?, ?, ?, ?, ?) using ttl ?");

        insertMainThreadProfile = session.prepare("insert into trace_main_thread_profile"
                + " (agent_id, trace_id, profile) values (?, ?, ?) using ttl ?");

        insertAuxThreadProfile = session.prepare("insert into trace_aux_thread_profile"
                + " (agent_id, trace_id, profile) values (?, ?, ?) using ttl ?");

        readCheck = session.prepare("select trace_id from trace_check where agent_rollup = ?"
                + " and agent_id = ? and trace_id = ?");

        readOverallSlowPoint = session.prepare("select agent_id, trace_id, capture_time,"
                + " duration_nanos, partial, error, headline, user, attributes"
                + " from trace_tt_slow_point where agent_rollup = ? and transaction_type = ?"
                + " and capture_time > ? and capture_time <= ?");

        readTransactionSlowPoint = session.prepare("select agent_id, trace_id, capture_time,"
                + " duration_nanos, partial, error, headline, user, attributes"
                + " from trace_tn_slow_point where agent_rollup = ? and transaction_type = ?"
                + " and transaction_name = ? and capture_time > ? and capture_time <= ?");

        readOverallErrorPoint = session.prepare("select agent_id, trace_id, capture_time,"
                + " duration_nanos, partial, error_message, headline, user, attributes"
                + " from trace_tt_error_point where agent_rollup = ? and transaction_type = ?"
                + " and capture_time > ? and capture_time <= ?");

        readTransactionErrorPoint = session.prepare("select agent_id, trace_id, capture_time,"
                + " duration_nanos, partial, error_message, headline, user, attributes"
                + " from trace_tn_error_point where agent_rollup = ? and transaction_type = ?"
                + " and transaction_name = ? and capture_time > ? and capture_time <= ?");

        readOverallErrorMessage = session.prepare("select capture_time, error_message"
                + " from trace_tt_error_message where agent_rollup = ? and transaction_type = ?"
                + " and capture_time > ? and capture_time <= ?");

        readTransactionErrorMessage = session.prepare("select capture_time, error_message"
                + " from trace_tn_error_message where agent_rollup = ? and transaction_type = ?"
                + " and transaction_name = ? and capture_time > ? and capture_time <= ?");

        readHeader = session
                .prepare("select header from trace_header where agent_id = ? and trace_id = ?");

        readEntries = session.prepare("select depth, start_offset_nanos, duration_nanos,"
                + " active, message, shared_query_text_index, query_message_prefix,"
                + " query_message_suffix, detail, location_stack_trace, error from trace_entry"
                + " where agent_id = ? and trace_id = ?");

        readSharedQueryTexts = session.prepare("select truncated_text, truncated_end_text,"
                + " full_text_sha1 from trace_shared_query_text where agent_id = ?"
                + " and trace_id = ?");

        readMainThreadProfile = session.prepare("select profile from trace_main_thread_profile"
                + " where agent_id = ? and trace_id = ?");

        readAuxThreadProfile = session.prepare("select profile from trace_aux_thread_profile"
                + " where agent_id = ? and trace_id = ?");

        deletePartialOverallSlowPoint = session.prepare("delete from trace_tt_slow_point"
                + " where agent_rollup = ? and transaction_type = ? and capture_time = ?"
                + " and agent_id = ? and trace_id = ?");

        deletePartialTransactionSlowPoint = session.prepare("delete from trace_tn_slow_point"
                + " where agent_rollup = ? and transaction_type = ? and transaction_name = ?"
                + " and capture_time = ? and agent_id = ? and trace_id = ?");

        deletePartialOverallSlowCount = session.prepare("delete from trace_tt_slow_count"
                + " where agent_rollup = ? and transaction_type = ? and capture_time = ?"
                + " and agent_id = ? and trace_id = ?");

        deletePartialTransactionSlowCount = session.prepare("delete from trace_tn_slow_count"
                + " where agent_rollup = ? and transaction_type = ? and transaction_name = ?"
                + " and capture_time = ? and agent_id = ? and trace_id = ?");
    }

    public void store(String agentId, Trace trace) throws Exception {
        String traceId = trace.getId();
        Trace.Header priorHeader = trace.getUpdate() ? readHeader(agentId, traceId) : null;
        Trace.Header header = trace.getHeader();

        List<String> agentRollupIds = agentDao.readAgentRollupIds(agentId);

        List<ResultSetFuture> futures = Lists.newArrayList();

        List<Trace.SharedQueryText> sharedQueryTexts = Lists.newArrayList();
        for (Trace.SharedQueryText sharedQueryText : trace.getSharedQueryTextList()) {
            String fullTextSha1 = sharedQueryText.getFullTextSha1();
            if (fullTextSha1.isEmpty()) {
                String fullText = sharedQueryText.getFullText();
                if (fullText.length() > 2 * StorageConfig.TRACE_QUERY_TEXT_TRUNCATE) {
                    fullTextSha1 = Hashing.sha1().hashString(fullText, Charsets.UTF_8).toString();
                    futures.addAll(fullQueryTextDao.store(agentId, fullTextSha1, fullText));
                    for (int i = 1; i < agentRollupIds.size(); i++) {
                        futures.addAll(
                                fullQueryTextDao.updateCheckTTL(agentRollupIds.get(i),
                                        fullTextSha1));
                    }
                    sharedQueryTexts.add(Trace.SharedQueryText.newBuilder()
                            .setTruncatedText(
                                    fullText.substring(0, StorageConfig.TRACE_QUERY_TEXT_TRUNCATE))
                            .setTruncatedEndText(fullText.substring(
                                    fullText.length() - StorageConfig.TRACE_QUERY_TEXT_TRUNCATE,
                                    fullText.length()))
                            .setFullTextSha1(fullTextSha1)
                            .build());
                } else {
                    sharedQueryTexts.add(sharedQueryText);
                }
            } else {
                futures.addAll(fullQueryTextDao.updateTTL(agentId, fullTextSha1));
                for (int i = 1; i < agentRollupIds.size(); i++) {
                    futures.addAll(
                            fullQueryTextDao.updateCheckTTL(agentRollupIds.get(i), fullTextSha1));
                }
                sharedQueryTexts.add(sharedQueryText);
            }
        }

        // wait for success before proceeding in order to ensure cannot end up with orphaned
        // fullTextSha1
        MoreFutures.waitForAll(futures);
        futures.clear();

        int adjustedTTL = AggregateDao.getAdjustedTTL(getTTL(), header.getCaptureTime(), clock);
        for (String agentRollupId : agentRollupIds) {

            if (!agentRollupId.equals(agentId)) {
                BoundStatement boundStatement = insertCheck.bind();
                int i = 0;
                boundStatement.setString(i++, agentRollupId);
                boundStatement.setString(i++, agentId);
                boundStatement.setString(i++, traceId);
                boundStatement.setInt(i++, adjustedTTL);
                futures.add(session.executeAsync(boundStatement));
            }

            if (header.getSlow()) {
                BoundStatement boundStatement = insertOverallSlowPoint.bind();
                bindSlowPoint(boundStatement, agentRollupId, agentId, traceId, header, adjustedTTL,
                        true);
                futures.add(session.executeAsync(boundStatement));

                boundStatement = insertTransactionSlowPoint.bind();
                bindSlowPoint(boundStatement, agentRollupId, agentId, traceId, header, adjustedTTL,
                        false);
                futures.add(session.executeAsync(boundStatement));

                boundStatement = insertOverallSlowCount.bind();
                bindCount(boundStatement, agentRollupId, agentId, traceId, header, adjustedTTL,
                        true);
                futures.add(session.executeAsync(boundStatement));

                boundStatement = insertTransactionSlowCount.bind();
                bindCount(boundStatement, agentRollupId, agentId, traceId, header, adjustedTTL,
                        false);
                futures.add(session.executeAsync(boundStatement));

                if (priorHeader != null) {
                    boundStatement = deletePartialOverallSlowPoint.bind();
                    bind(boundStatement, agentRollupId, agentId, traceId, priorHeader, true);
                    futures.add(session.executeAsync(boundStatement));

                    boundStatement = deletePartialTransactionSlowPoint.bind();
                    bind(boundStatement, agentRollupId, agentId, traceId, priorHeader, false);
                    futures.add(session.executeAsync(boundStatement));

                    boundStatement = deletePartialOverallSlowCount.bind();
                    bind(boundStatement, agentRollupId, agentId, traceId, priorHeader, true);
                    futures.add(session.executeAsync(boundStatement));

                    boundStatement = deletePartialTransactionSlowCount.bind();
                    bind(boundStatement, agentRollupId, agentId, traceId, priorHeader, false);
                    futures.add(session.executeAsync(boundStatement));
                }
            }
            // seems unnecessary to insert error info for partial traces
            // and this avoids having to clean up partial trace data when trace is complete
            if (header.hasError() && !header.getPartial()) {
                BoundStatement boundStatement = insertOverallErrorMessage.bind();
                bindErrorMessage(boundStatement, agentRollupId, agentId, traceId, header,
                        adjustedTTL, true);
                futures.add(session.executeAsync(boundStatement));

                boundStatement = insertTransactionErrorMessage.bind();
                bindErrorMessage(boundStatement, agentRollupId, agentId, traceId, header,
                        adjustedTTL, false);
                futures.add(session.executeAsync(boundStatement));

                boundStatement = insertOverallErrorPoint.bind();
                bindErrorPoint(boundStatement, agentRollupId, agentId, traceId, header, adjustedTTL,
                        true);
                futures.add(session.executeAsync(boundStatement));

                boundStatement = insertTransactionErrorPoint.bind();
                bindErrorPoint(boundStatement, agentRollupId, agentId, traceId, header, adjustedTTL,
                        false);
                futures.add(session.executeAsync(boundStatement));

                boundStatement = insertOverallErrorCount.bind();
                bindCount(boundStatement, agentRollupId, agentId, traceId, header, adjustedTTL,
                        true);
                futures.add(session.executeAsync(boundStatement));

                boundStatement = insertTransactionErrorCount.bind();
                bindCount(boundStatement, agentRollupId, agentId, traceId, header, adjustedTTL,
                        false);
                futures.add(session.executeAsync(boundStatement));
            }
            for (Trace.Attribute attributeName : header.getAttributeList()) {
                traceAttributeNameDao.store(agentRollupId,
                        header.getTransactionType(), attributeName.getName(), futures);
            }
        }

        BoundStatement boundStatement = insertHeader.bind();
        int i = 0;
        boundStatement.setString(i++, agentId);
        boundStatement.setString(i++, traceId);
        boundStatement.setBytes(i++, ByteBuffer.wrap(header.toByteArray()));
        boundStatement.setInt(i++, adjustedTTL);
        futures.add(session.executeAsync(boundStatement));

        int index = 0;
        for (Trace.Entry entry : trace.getEntryList()) {
            boundStatement = insertEntry.bind();
            i = 0;
            boundStatement.setString(i++, agentId);
            boundStatement.setString(i++, traceId);
            boundStatement.setInt(i++, index++);
            boundStatement.setInt(i++, entry.getDepth());
            boundStatement.setLong(i++, entry.getStartOffsetNanos());
            boundStatement.setLong(i++, entry.getDurationNanos());
            boundStatement.setBool(i++, entry.getActive());
            if (entry.hasQueryEntryMessage()) {
                boundStatement.setToNull(i++);
                boundStatement.setInt(i++, entry.getQueryEntryMessage().getSharedQueryTextIndex());
                boundStatement.setString(i++,
                        Strings.emptyToNull(entry.getQueryEntryMessage().getPrefix()));
                boundStatement.setString(i++,
                        Strings.emptyToNull(entry.getQueryEntryMessage().getSuffix()));
            } else {
                // message is empty for trace entries added using addErrorEntry()
                boundStatement.setString(i++, Strings.emptyToNull(entry.getMessage()));
                boundStatement.setToNull(i++);
                boundStatement.setToNull(i++);
                boundStatement.setToNull(i++);
            }
            List<Trace.DetailEntry> detailEntries = entry.getDetailEntryList();
            if (detailEntries.isEmpty()) {
                boundStatement.setToNull(i++);
            } else {
                boundStatement.setBytes(i++, Messages.toByteBuffer(detailEntries));
            }
            List<StackTraceElement> location = entry.getLocationStackTraceElementList();
            if (location.isEmpty()) {
                boundStatement.setToNull(i++);
            } else {
                boundStatement.setBytes(i++, Messages.toByteBuffer(location));
            }
            if (entry.hasError()) {
                boundStatement.setBytes(i++, ByteBuffer.wrap(entry.getError().toByteArray()));
            } else {
                boundStatement.setToNull(i++);
            }
            boundStatement.setInt(i++, adjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }

        index = 0;
        for (Trace.SharedQueryText sharedQueryText : sharedQueryTexts) {
            boundStatement = insertSharedQueryText.bind();
            i = 0;
            boundStatement.setString(i++, agentId);
            boundStatement.setString(i++, traceId);
            boundStatement.setInt(i++, index++);
            String fullText = sharedQueryText.getFullText();
            if (fullText.isEmpty()) {
                boundStatement.setString(i++, sharedQueryText.getTruncatedText());
                boundStatement.setString(i++, sharedQueryText.getTruncatedEndText());
                boundStatement.setString(i++, sharedQueryText.getFullTextSha1());
            } else {
                boundStatement.setString(i++, fullText);
                boundStatement.setToNull(i++);
                boundStatement.setToNull(i++);
            }
            boundStatement.setInt(i++, adjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }

        if (trace.hasMainThreadProfile()) {
            boundStatement = insertMainThreadProfile.bind();
            bindThreadProfile(boundStatement, agentId, traceId, trace.getMainThreadProfile(),
                    adjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }

        if (trace.hasAuxThreadProfile()) {
            boundStatement = insertAuxThreadProfile.bind();
            bindThreadProfile(boundStatement, agentId, traceId, trace.getAuxThreadProfile(),
                    adjustedTTL);
            futures.add(session.executeAsync(boundStatement));
        }
        futures.addAll(transactionTypeDao.store(agentRollupIds, header.getTransactionType()));
        MoreFutures.waitForAll(futures);
    }

    @Override
    public Result<TracePoint> readSlowPoints(String agentRollupId, TraceQuery query,
            TracePointFilter filter, int limit) throws IOException {
        BoundStatement boundStatement;
        String transactionName = query.transactionName();
        if (transactionName == null) {
            boundStatement = readOverallSlowPoint.bind();
            bindTraceQuery(boundStatement, agentRollupId, query, true);
        } else {
            boundStatement = readTransactionSlowPoint.bind();
            bindTraceQuery(boundStatement, agentRollupId, query, false);
        }
        ResultSet results = session.execute(boundStatement);
        return processPoints(results, filter, limit, false);
    }

    @Override
    public Result<TracePoint> readErrorPoints(String agentRollupId, TraceQuery query,
            TracePointFilter filter, int limit) throws IOException {
        BoundStatement boundStatement;
        String transactionName = query.transactionName();
        if (transactionName == null) {
            boundStatement = readOverallErrorPoint.bind();
            bindTraceQuery(boundStatement, agentRollupId, query, true);
        } else {
            boundStatement = readTransactionErrorPoint.bind();
            bindTraceQuery(boundStatement, agentRollupId, query, false);
        }
        ResultSet results = session.execute(boundStatement);
        return processPoints(results, filter, limit, true);
    }

    @Override
    public long readSlowCount(String agentRollupId, TraceQuery query) {
        String transactionName = query.transactionName();
        if (transactionName == null) {
            ResultSet results = session.execute(
                    "select count(*) from trace_tt_slow_count where agent_rollup = ?"
                            + " and transaction_type = ? and capture_time > ?"
                            + " and capture_time <= ?",
                    agentRollupId, query.transactionType(), query.from(), query.to());
            return results.one().getLong(0);
        } else {
            ResultSet results = session.execute(
                    "select count(*) from trace_tn_slow_count where agent_rollup = ?"
                            + " and transaction_type = ? and transaction_name = ?"
                            + " and capture_time > ? and capture_time <= ?",
                    agentRollupId, query.transactionType(), transactionName, query.from(),
                    query.to());
            return results.one().getLong(0);
        }
    }

    @Override
    public long readErrorCount(String agentRollupId, TraceQuery query) {
        String transactionName = query.transactionName();
        if (transactionName == null) {
            ResultSet results = session.execute(
                    "select count(*) from trace_tt_error_count where agent_rollup = ?"
                            + " and transaction_type = ? and capture_time > ?"
                            + " and capture_time <= ?",
                    agentRollupId, query.transactionType(), query.from(), query.to());
            return results.one().getLong(0);
        } else {
            ResultSet results = session.execute(
                    "select count(*) from trace_tn_error_count where agent_rollup = ?"
                            + " and transaction_type = ? and transaction_name = ?"
                            + " and capture_time > ? and capture_time <= ?",
                    agentRollupId, query.transactionType(), transactionName, query.from(),
                    query.to());
            return results.one().getLong(0);
        }
    }

    @Override
    public ErrorMessageResult readErrorMessages(String agentRollupId, TraceQuery query,
            ErrorMessageFilter filter, long resolutionMillis, int limit) throws Exception {
        BoundStatement boundStatement;
        String transactionName = query.transactionName();
        if (transactionName == null) {
            boundStatement = readOverallErrorMessage.bind();
            bindTraceQuery(boundStatement, agentRollupId, query, true);
        } else {
            boundStatement = readTransactionErrorMessage.bind();
            bindTraceQuery(boundStatement, agentRollupId, query, false);
        }
        ResultSet results = session.execute(boundStatement);
        // rows are already in order by captureTime, so saving sort step by using linked hash map
        Map<Long, MutableLong> pointCounts = Maps.newLinkedHashMap();
        Map<String, MutableLong> messageCounts = Maps.newHashMap();
        for (Row row : results) {
            long captureTime = checkNotNull(row.getTimestamp(0)).getTime();
            String errorMessage = checkNotNull(row.getString(1));
            if (!matches(filter, errorMessage)) {
                continue;
            }
            long rollupCaptureTime = Utils.getRollupCaptureTime(captureTime, resolutionMillis);
            pointCounts.computeIfAbsent(rollupCaptureTime, k -> new MutableLong()).increment();
            messageCounts.computeIfAbsent(errorMessage, k -> new MutableLong()).increment();
        }
        List<ErrorMessagePoint> points = pointCounts.entrySet().stream()
                .map(e -> ImmutableErrorMessagePoint.of(e.getKey(), e.getValue().value))
                .sorted(Comparator.comparingLong(ErrorMessagePoint::captureTime))
                // explicit type on this line is needed for Checker Framework
                // see https://github.com/typetools/checker-framework/issues/531
                .collect(Collectors.<ErrorMessagePoint>toList());
        List<ErrorMessageCount> counts = messageCounts.entrySet().stream()
                .map(e -> ImmutableErrorMessageCount.of(e.getKey(), e.getValue().value))
                // points above are already ordered by cassandra, but need to reverse sort counts
                .sorted(Comparator.comparing(ErrorMessageCount::count).reversed())
                // explicit type on this line is needed for Checker Framework
                // see https://github.com/typetools/checker-framework/issues/531
                .collect(Collectors.<ErrorMessageCount>toList());

        if (counts.size() <= limit) {
            return ImmutableErrorMessageResult.builder()
                    .addAllPoints(points)
                    .counts(new Result<>(counts, false))
                    .build();
        } else {
            return ImmutableErrorMessageResult.builder()
                    .addAllPoints(points)
                    .counts(new Result<>(counts.subList(0, limit), true))
                    .build();
        }
    }

    @Override
    public @Nullable HeaderPlus readHeaderPlus(String agentRollupId, String agentId, String traceId)
            throws InvalidProtocolBufferException {
        checkValidAgentIdForRequest(agentRollupId, agentId, traceId);
        Trace.Header header = readHeader(agentId, traceId);
        if (header == null) {
            return null;
        }
        Existence entriesExistence = header.getEntryCount() == 0 ? Existence.NO : Existence.YES;
        Existence profileExistence = header.getMainThreadProfileSampleCount() == 0
                && header.getAuxThreadProfileSampleCount() == 0 ? Existence.NO : Existence.YES;
        return ImmutableHeaderPlus.of(header, entriesExistence, profileExistence);
    }

    @Override
    public Entries readEntries(String agentRollupId, String agentId, String traceId)
            throws IOException {
        checkValidAgentIdForRequest(agentRollupId, agentId, traceId);
        return ImmutableEntries.builder()
                .addAllEntries(readEntriesInternal(agentId, traceId))
                .addAllSharedQueryTexts(readSharedQueryTexts(agentId, traceId))
                .build();
    }

    // since this is only used by export, SharedQueryTexts are always returned with fullTrace
    // (never with truncatedText/truncatedEndText/fullTraceSha1) @Override
    @Override
    public Entries readEntriesForExport(String agentRollupId, String agentId, String traceId)
            throws IOException {
        checkValidAgentIdForRequest(agentRollupId, agentId, traceId);
        ImmutableEntries.Builder entries = ImmutableEntries.builder()
                .addAllEntries(readEntriesInternal(agentId, traceId));
        List<Trace.SharedQueryText> sharedQueryTexts = Lists.newArrayList();
        for (Trace.SharedQueryText sharedQueryText : readSharedQueryTexts(agentId, traceId)) {
            String fullTextSha1 = sharedQueryText.getFullTextSha1();
            if (fullTextSha1.isEmpty()) {
                sharedQueryTexts.add(sharedQueryText);
            } else {
                String fullText = fullQueryTextDao.getFullText(agentId, fullTextSha1);
                if (fullText == null) {
                    sharedQueryTexts.add(Trace.SharedQueryText.newBuilder()
                            .setFullText(sharedQueryText.getTruncatedText()
                                    + " ... [full query text has expired] ... "
                                    + sharedQueryText.getTruncatedEndText())
                            .build());
                } else {
                    sharedQueryTexts.add(Trace.SharedQueryText.newBuilder()
                            .setFullText(fullText)
                            .build());
                }
            }
        }
        return entries.addAllSharedQueryTexts(sharedQueryTexts)
                .build();
    }

    @Override
    public @Nullable Profile readMainThreadProfile(String agentRollupId, String agentId,
            String traceId) throws InvalidProtocolBufferException {
        checkValidAgentIdForRequest(agentRollupId, agentId, traceId);
        BoundStatement boundStatement = readMainThreadProfile.bind();
        boundStatement.setString(0, agentId);
        boundStatement.setString(1, traceId);
        ResultSet results = session.execute(boundStatement);
        Row row = results.one();
        if (row == null) {
            return null;
        }
        ByteBuffer bytes = checkNotNull(row.getBytes(0));
        return Profile.parseFrom(ByteString.copyFrom(bytes));
    }

    @Override
    public @Nullable Profile readAuxThreadProfile(String agentRollupId, String agentId,
            String traceId) throws InvalidProtocolBufferException {
        checkValidAgentIdForRequest(agentRollupId, agentId, traceId);
        BoundStatement boundStatement = readAuxThreadProfile.bind();
        boundStatement.setString(0, agentId);
        boundStatement.setString(1, traceId);
        ResultSet results = session.execute(boundStatement);
        Row row = results.one();
        if (row == null) {
            return null;
        }
        ByteBuffer bytes = checkNotNull(row.getBytes(0));
        return Profile.parseFrom(ByteString.copyFrom(bytes));
    }

    private void checkValidAgentIdForRequest(String agentRollupId, String agentId, String traceId) {
        if (agentId.equals(agentRollupId)) {
            return;
        }
        BoundStatement boundStatement = readCheck.bind();
        int i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, agentId);
        boundStatement.setString(i++, traceId);
        if (session.execute(boundStatement).isExhausted()) {
            throw new IllegalArgumentException("Agent " + agentId + " was not a child of rollup "
                    + agentRollupId + " at the time of trace " + traceId);
        }
    }

    private @Nullable Trace.Header readHeader(String agentId, String traceId)
            throws InvalidProtocolBufferException {
        BoundStatement boundStatement = readHeader.bind();
        boundStatement.setString(0, agentId);
        boundStatement.setString(1, traceId);
        ResultSet results = session.execute(boundStatement);
        Row row = results.one();
        if (row == null) {
            return null;
        }
        ByteBuffer bytes = checkNotNull(row.getBytes(0));
        return Trace.Header.parseFrom(ByteString.copyFrom(bytes));
    }

    private List<Trace.Entry> readEntriesInternal(String agentId, String traceId)
            throws IOException {
        BoundStatement boundStatement = readEntries.bind();
        boundStatement.setString(0, agentId);
        boundStatement.setString(1, traceId);
        ResultSet results = session.execute(boundStatement);
        List<Trace.Entry> entries = Lists.newArrayList();
        while (!results.isExhausted()) {
            Row row = results.one();
            int i = 0;
            Trace.Entry.Builder entry = Trace.Entry.newBuilder()
                    .setDepth(row.getInt(i++))
                    .setStartOffsetNanos(row.getLong(i++))
                    .setDurationNanos(row.getLong(i++))
                    .setActive(row.getBool(i++));
            if (row.isNull(i + 1)) { // shared_query_text_index
                // message is null for trace entries added using addErrorEntry()
                entry.setMessage(Strings.nullToEmpty(row.getString(i++)));
                i++; // shared_query_text_index
                i++; // query_message_prefix
                i++; // query_message_suffix
            } else {
                i++; // message
                Trace.QueryEntryMessage queryEntryMessage = Trace.QueryEntryMessage.newBuilder()
                        .setSharedQueryTextIndex(row.getInt(i++))
                        .setPrefix(Strings.nullToEmpty(row.getString(i++)))
                        .setSuffix(Strings.nullToEmpty(row.getString(i++)))
                        .build();
                entry.setQueryEntryMessage(queryEntryMessage);
            }
            ByteBuffer detailBytes = row.getBytes(i++);
            if (detailBytes != null) {
                entry.addAllDetailEntry(
                        Messages.parseDelimitedFrom(detailBytes, Trace.DetailEntry.parser()));
            }
            ByteBuffer locationBytes = row.getBytes(i++);
            if (locationBytes != null) {
                entry.addAllLocationStackTraceElement(Messages.parseDelimitedFrom(locationBytes,
                        Proto.StackTraceElement.parser()));
            }
            ByteBuffer errorBytes = row.getBytes(i++);
            if (errorBytes != null) {
                entry.setError(Trace.Error.parseFrom(ByteString.copyFrom(errorBytes)));
            }
            entries.add(entry.build());
        }
        return entries;
    }

    private List<Trace.SharedQueryText> readSharedQueryTexts(String agentId, String traceId)
            throws IOException {
        BoundStatement boundStatement = readSharedQueryTexts.bind();
        boundStatement.setString(0, agentId);
        boundStatement.setString(1, traceId);
        ResultSet results = session.execute(boundStatement);
        List<Trace.SharedQueryText> sharedQueryTexts = Lists.newArrayList();
        while (!results.isExhausted()) {
            Row row = results.one();
            int i = 0;
            String truncatedText = checkNotNull(row.getString(i++));
            String truncatedEndText = row.getString(i++);
            String fullTextSha1 = row.getString(i++);
            Trace.SharedQueryText.Builder sharedQueryText = Trace.SharedQueryText.newBuilder();
            if (fullTextSha1 == null) {
                sharedQueryText.setFullText(truncatedText);
            } else {
                sharedQueryText.setFullTextSha1(fullTextSha1)
                        .setTruncatedText(truncatedText)
                        .setTruncatedEndText(checkNotNull(truncatedEndText));
            }
            sharedQueryTexts.add(sharedQueryText.build());
        }
        return sharedQueryTexts;
    }

    private int getTTL() throws Exception {
        return Ints.saturatedCast(
                HOURS.toSeconds(configRepository.getStorageConfig().traceExpirationHours()));
    }

    private static void bindSlowPoint(BoundStatement boundStatement, String agentRollupId,
            String agentId, String traceId, Trace.Header header, int adjustedTTL, boolean overall)
            throws IOException {
        int i = bind(boundStatement, agentRollupId, agentId, traceId, header, overall);
        boundStatement.setLong(i++, header.getDurationNanos());
        boundStatement.setBool(i++, header.getPartial());
        boundStatement.setBool(i++, header.hasError());
        boundStatement.setString(i++, header.getHeadline());
        boundStatement.setString(i++, Strings.emptyToNull(header.getUser()));
        List<Trace.Attribute> attributes = header.getAttributeList();
        if (attributes.isEmpty()) {
            boundStatement.setToNull(i++);
        } else {
            boundStatement.setBytes(i++, Messages.toByteBuffer(attributes));
        }
        boundStatement.setInt(i++, adjustedTTL);
    }

    private static void bindCount(BoundStatement boundStatement, String agentRollupId,
            String agentId, String traceId, Trace.Header header, int adjustedTTL, boolean overall) {
        int i = bind(boundStatement, agentRollupId, agentId, traceId, header, overall);
        boundStatement.setInt(i++, adjustedTTL);
    }

    private static void bindErrorMessage(BoundStatement boundStatement, String agentRollupId,
            String agentId, String traceId, Trace.Header header, int adjustedTTL, boolean overall) {
        int i = bind(boundStatement, agentRollupId, agentId, traceId, header, overall);
        boundStatement.setString(i++, header.getError().getMessage());
        boundStatement.setInt(i++, adjustedTTL);
    }

    private static void bindErrorPoint(BoundStatement boundStatement, String agentRollupId,
            String agentId, String traceId, Trace.Header header, int adjustedTTL, boolean overall)
            throws IOException {
        int i = bind(boundStatement, agentRollupId, agentId, traceId, header, overall);
        boundStatement.setLong(i++, header.getDurationNanos());
        boundStatement.setBool(i++, header.getPartial());
        boundStatement.setString(i++, header.getError().getMessage());
        boundStatement.setString(i++, header.getHeadline());
        boundStatement.setString(i++, Strings.emptyToNull(header.getUser()));
        List<Trace.Attribute> attributes = header.getAttributeList();
        if (attributes.isEmpty()) {
            boundStatement.setToNull(i++);
        } else {
            boundStatement.setBytes(i++, Messages.toByteBuffer(attributes));
        }
        boundStatement.setInt(i++, adjustedTTL);
    }

    private static int bind(BoundStatement boundStatement, String agentRollupId, String agentId,
            String traceId, Trace.Header header, boolean overall) {
        int i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, header.getTransactionType());
        if (!overall) {
            boundStatement.setString(i++, header.getTransactionName());
        }
        boundStatement.setTimestamp(i++, new Date(header.getCaptureTime()));
        boundStatement.setString(i++, agentId);
        boundStatement.setString(i++, traceId);
        return i;
    }

    private static void bindThreadProfile(BoundStatement boundStatement, String agentId,
            String traceId, Profile profile, int adjustedTTL) {
        int i = 0;
        boundStatement.setString(i++, agentId);
        boundStatement.setString(i++, traceId);
        boundStatement.setBytes(i++, ByteBuffer.wrap(profile.toByteArray()));
        boundStatement.setInt(i++, adjustedTTL);
    }

    private static void bindTraceQuery(BoundStatement boundStatement, String agentRollupId,
            TraceQuery query, boolean overall) {
        int i = 0;
        boundStatement.setString(i++, agentRollupId);
        boundStatement.setString(i++, query.transactionType());
        String transactionName = query.transactionName();
        if (!overall) {
            boundStatement.setString(i++, transactionName);
        }
        boundStatement.setTimestamp(i++, new Date(query.from()));
        boundStatement.setTimestamp(i++, new Date(query.to()));
    }

    private static Result<TracePoint> processPoints(ResultSet results, TracePointFilter filter,
            int limit, boolean errorPoints) throws IOException {
        List<TracePoint> tracePoints = Lists.newArrayList();
        for (Row row : results) {
            int i = 0;
            String agentId = checkNotNull(row.getString(i++));
            String traceId = checkNotNull(row.getString(i++));
            long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
            long durationNanos = row.getLong(i++);
            boolean partial = row.getBool(i++);
            boolean error = errorPoints ? true : row.getBool(i++);
            // error points are defined by having an error message, so safe to checkNotNull
            String errorMessage = errorPoints ? checkNotNull(row.getString(i++)) : "";
            // headline is null for data inserted prior to 0.9.7
            String headline = Strings.nullToEmpty(row.getString(i++));
            String user = Strings.nullToEmpty(row.getString(i++));
            ByteBuffer attributeBytes = row.getBytes(i++);
            List<Trace.Attribute> attrs =
                    Messages.parseDelimitedFrom(attributeBytes, Trace.Attribute.parser());
            Map<String, List<String>> attributes = attrs.stream().collect(
                    Collectors.toMap(Trace.Attribute::getName, Trace.Attribute::getValueList));
            if (filter.matchesHeadline(headline)
                    && filter.matchesError(errorMessage)
                    && filter.matchesUser(user)
                    && filter.matchesAttributes(attributes)) {
                tracePoints.add(ImmutableTracePoint.builder()
                        .agentId(agentId)
                        .traceId(traceId)
                        .captureTime(captureTime)
                        .durationNanos(durationNanos)
                        .partial(partial)
                        .error(error)
                        .build());
            }
        }
        // remove duplicates (partially stored traces) since there is (small) window between updated
        // insert (with new capture time) and the delete of prior insert (with prior capture time)
        Set<TraceKey> traceKeys = Sets.newHashSet();
        ListIterator<TracePoint> i = tracePoints.listIterator(tracePoints.size());
        while (i.hasPrevious()) {
            TracePoint trace = i.previous();
            TraceKey traceKey = ImmutableTraceKey.of(trace.agentId(), trace.traceId());
            if (!traceKeys.add(traceKey)) {
                i.remove();
            }
        }
        // apply limit and re-sort if needed
        if (tracePoints.size() > limit) {
            tracePoints = tracePoints.stream()
                    .sorted(Comparator.comparingLong(TracePoint::durationNanos).reversed())
                    .limit(limit)
                    .sorted(Comparator.comparingLong(TracePoint::captureTime))
                    // explicit type on this line is needed for Checker Framework
                    // see https://github.com/typetools/checker-framework/issues/531
                    .collect(Collectors.<TracePoint>toList());
            return new Result<>(tracePoints, true);
        } else {
            return new Result<>(tracePoints, false);
        }
    }

    private static boolean matches(ErrorMessageFilter filter, String errorMessage) {
        String upper = errorMessage.toUpperCase(Locale.ENGLISH);
        for (String include : filter.includes()) {
            if (!upper.contains(include.toUpperCase(Locale.ENGLISH))) {
                return false;
            }
        }
        for (String exclude : filter.excludes()) {
            if (upper.contains(exclude.toUpperCase(Locale.ENGLISH))) {
                return false;
            }
        }
        return true;
    }

    @Value.Immutable
    @Styles.AllParameters
    interface TraceKey {
        String agentId();
        String traceId();
    }

    private static class MutableLong {
        private long value;
        private void increment() {
            value++;
        }
    }
}