TimeSeriesRepositoryImpl.java

// =================================================================================================
// Copyright 2011 Twitter, Inc.
// -------------------------------------------------------------------------------------------------
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this work except in compliance with the License.
// You may obtain a copy of the License in the LICENSE file, or at:
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// =================================================================================================

package com.twitter.common.stats;

import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import java.util.logging.Logger;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.inject.Inject;
import com.google.inject.name.Named;

import com.twitter.common.application.ShutdownRegistry;
import com.twitter.common.base.Command;
import com.twitter.common.collections.BoundedQueue;
import com.twitter.common.quantity.Amount;
import com.twitter.common.quantity.Time;
import com.twitter.common.util.Clock;

import static com.google.common.base.Preconditions.checkNotNull;

/**
 * A simple in-memory repository for exported variables.
 *
 * @author John Sirois
 */
public class TimeSeriesRepositoryImpl implements TimeSeriesRepository {

  private static final Logger LOG = Logger.getLogger(TimeSeriesRepositoryImpl.class.getName());

  /**
   * {@literal @Named} binding key for the sampling period.
   */
  public static final String SAMPLE_PERIOD =
      "com.twitter.common.stats.TimeSeriesRepositoryImpl.SAMPLE_PERIOD";

  /**
   * {@literal @Named} binding key for the maximum number of retained samples.
   */
  public static final String SAMPLE_RETENTION_PERIOD =
      "com.twitter.common.stats.TimeSeriesRepositoryImpl.SAMPLE_RETENTION_PERIOD";

  private final SlidingStats scrapeDuration = new SlidingStats("variable_scrape", "micros");

  // We store TimeSeriesImpl, which allows us to add samples.
  private final LoadingCache<String, TimeSeriesImpl> timeSeries;
  private final BoundedQueue<Number> timestamps;

  private final StatRegistry statRegistry;
  private final Amount<Long, Time> samplePeriod;
  private final int retainedSampleLimit;

  @Inject
  public TimeSeriesRepositoryImpl(
      StatRegistry statRegistry,
      @Named(SAMPLE_PERIOD) Amount<Long, Time> samplePeriod,
      @Named(SAMPLE_RETENTION_PERIOD) final Amount<Long, Time> retentionPeriod) {
    this.statRegistry = checkNotNull(statRegistry);
    this.samplePeriod = checkNotNull(samplePeriod);
    Preconditions.checkArgument(samplePeriod.getValue() > 0, "Sample period must be positive.");
    checkNotNull(retentionPeriod);
    Preconditions.checkArgument(retentionPeriod.getValue() > 0,
        "Sample retention period must be positive.");

    retainedSampleLimit = (int) (retentionPeriod.as(Time.SECONDS) / samplePeriod.as(Time.SECONDS));
    Preconditions.checkArgument(retainedSampleLimit > 0,
        "Sample retention period must be greater than sample period.");

    timeSeries = CacheBuilder.newBuilder().build(
        new CacheLoader<String, TimeSeriesImpl>() {
          @Override public TimeSeriesImpl load(final String name) {
            TimeSeriesImpl timeSeries = new TimeSeriesImpl(name);

            // Backfill so we have data for pre-accumulated timestamps.
            int numTimestamps = timestamps.size();
            if (numTimestamps != 0) {
              for (int i = 1; i < numTimestamps; i++) {
                timeSeries.addSample(0L);
              }
            }

            return timeSeries;
          }
        });

    timestamps = new BoundedQueue<Number>(retainedSampleLimit);
  }

  /**
   * Starts the variable sampler, which will fetch variables {@link Stats} on the given period.
   *
   */
  @Override
  public void start(ShutdownRegistry shutdownRegistry) {
    checkNotNull(shutdownRegistry);
    checkNotNull(samplePeriod);
    Preconditions.checkArgument(samplePeriod.getValue() > 0);

    final ScheduledExecutorService executor = new ScheduledThreadPoolExecutor(1 /* One thread. */,
        new ThreadFactoryBuilder().setNameFormat("VariableSampler-%d").setDaemon(true).build());

    final AtomicBoolean shouldSample = new AtomicBoolean(true);
    final Runnable sampler = new Runnable() {
      @Override public void run() {
        if (shouldSample.get()) {
          try {
            runSampler(Clock.SYSTEM_CLOCK);
          } catch (Exception e) {
            LOG.log(Level.SEVERE, "ignoring runSampler failure", e);
          }
        }
      }
    };

    executor.scheduleAtFixedRate(sampler, samplePeriod.getValue(), samplePeriod.getValue(),
        samplePeriod.getUnit().getTimeUnit());
    shutdownRegistry.addAction(new Command() {
      @Override
      public void execute() throws RuntimeException {
        shouldSample.set(false);
        executor.shutdown();
        LOG.info("Variable sampler shut down");
      }
    });
  }

  @VisibleForTesting
  synchronized void runSampler(Clock clock) {
    timestamps.add(clock.nowMillis());

    long startNanos = clock.nowNanos();
    for (RecordingStat<? extends Number> var : statRegistry.getStats()) {
      timeSeries.getUnchecked(var.getName()).addSample(var.sample());
    }
    scrapeDuration.accumulate(
        Amount.of(clock.nowNanos() - startNanos, Time.NANOSECONDS).as(Time.MICROSECONDS));
  }

  @Override
  public synchronized Set<String> getAvailableSeries() {
    return ImmutableSet.copyOf(timeSeries.asMap().keySet());
  }

  @Override
  public synchronized TimeSeries get(String name) {
    if (!timeSeries.asMap().containsKey(name)) return null;
    return timeSeries.getUnchecked(name);
  }

  @Override
  public synchronized Iterable<Number> getTimestamps() {
    return Iterables.unmodifiableIterable(timestamps);
  }

  private class TimeSeriesImpl implements TimeSeries {
    private final String name;
    private final BoundedQueue<Number> samples;

    TimeSeriesImpl(String name) {
      this.name = name;
      samples = new BoundedQueue<Number>(retainedSampleLimit);
    }

    @Override public String getName() {
      return name;
    }

    void addSample(Number value) {
      samples.add(value);
    }

    @Override public Iterable<Number> getSamples() {
      return Iterables.unmodifiableIterable(samples);
    }
  }
}