CosmosImpl.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with this
 * work for additional information regarding copyright ownership. The ASF
 * licenses this file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 *
 *  Copyright 2013 Josh Elser
 *
 */
package cosmos.impl;

import static com.google.common.base.Preconditions.checkNotNull;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchDeleter;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.lexicoder.ReverseLexicoder;
import org.apache.accumulo.core.client.lexicoder.StringLexicoder;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Stopwatch;
import com.google.common.collect.Sets;
import com.google.common.io.Closeables;

import cosmos.Cosmos;
import cosmos.UnexpectedStateException;
import cosmos.UnindexedColumnException;
import cosmos.accumulo.GroupByRowSuffixIterator;
import cosmos.accumulo.OrderFilter;
import cosmos.options.Defaults;
import cosmos.options.Index;
import cosmos.options.Order;
import cosmos.options.Paging;
import cosmos.records.Record;
import cosmos.records.impl.MultimapRecord;
import cosmos.records.values.RecordValue;
import cosmos.results.CloseableIterable;
import cosmos.results.Column;
import cosmos.results.PagedResults;
import cosmos.store.PersistedStores;
import cosmos.store.PersistedStores.State;
import cosmos.store.Store;
import cosmos.util.IndexHelper;
import cosmos.util.Single;

public class CosmosImpl implements Cosmos {
  private static final Logger log = LoggerFactory.getLogger(CosmosImpl.class);

  public static final long LOCK_SECS = 10;

  private final CuratorFramework curator;
  private final ReverseLexicoder<String> revLex = new ReverseLexicoder<String>(new StringLexicoder());

  public CosmosImpl(String zookeepers) {
    RetryPolicy retryPolicy = new ExponentialBackoffRetry(2000, 3);
    curator = CuratorFrameworkFactory.newClient(zookeepers, retryPolicy);
    curator.start();

    // TODO http://curator.incubator.apache.org/curator-recipes/shared-reentrant-lock.html
    // "Error handling: ... strongly recommended that you add a ConnectionStateListener and
    // watch for SUSPENDED and LOST state changes"

    // curator.getConnectionStateListenable().addListener(new ConnectionStateListener() {
    //
    // @Override
    // public void stateChanged(CuratorFramework client, ConnectionState newState) {
    //
    // }
    //
    // });
  }

  @Override
  public void close() {
    synchronized (curator) {
      CuratorFrameworkState state = curator.getState();

      // Stop unless we're already stopped
      if (!CuratorFrameworkState.STOPPED.equals(state)) {
        try {
          Closeables.close(curator, true);
        } catch (IOException e) {
          log.warn("Caught IOException closing Curator connection", e);
        }
      }
    }
  }

  /**
   * finalize is not guaranteed to be called, as such, care should be taken to ensure that {@link close} is called.
   */
  @Override
  public void finalize() throws IOException {
    this.close();
  }

  @Override
  public void register(Store id) throws TableNotFoundException, MutationsRejectedException, UnexpectedStateException {
    checkNotNull(id);

    Stopwatch sw = new Stopwatch().start();

    try {
      State s = PersistedStores.getState(id);

      if (!State.UNKNOWN.equals(s)) {
        UnexpectedStateException e = unexpectedState(id, State.UNKNOWN, s);
        log.error(e.getMessage());
        throw e;
      }

      State targetState = State.LOADING;

      log.debug("Setting state for {} from {} to {}", new Object[] {id, s, targetState});

      PersistedStores.setState(id, targetState);
    } finally {
      sw.stop();
      id.tracer().addTiming("Cosmos:register", sw.elapsed(TimeUnit.MILLISECONDS));
    }
  }

  @Override
  public void addResult(Store id, Record<?> queryResult) throws Exception {
    checkNotNull(queryResult);

    Stopwatch sw = new Stopwatch().start();
    try {
      addResults(id, Single.<Record<?>> create(queryResult));
    } finally {
      sw.stop();
      id.tracer().addTiming("Cosmos:addResult", sw.elapsed(TimeUnit.MILLISECONDS));
    }
  }

  @Override
  public void addResults(Store id, Iterable<? extends Record<?>> queryResults) throws Exception {
    checkNotNull(id);
    checkNotNull(queryResults);

    Stopwatch sw = new Stopwatch().start();
    try {
      State s = PersistedStores.getState(id);

      if (!State.LOADING.equals(s)) {
        // stopwatch closed in finally
        UnexpectedStateException e = unexpectedState(id, State.LOADING, s);
        log.error(e.getMessage());
        throw e;
      }

      InterProcessMutex lock = getMutex(id);

      // TODO We don't need to lock on multiple calls to addResults; however, we need to lock over adding the
      // new records to make sure a call to index() doesn't come in while we're processing a stale set of Columns to index
      boolean locked = false;
      int count = 1;

      if (id.lockOnUpdates()) {
        while (!locked && count < 4) {
          if (locked = lock.acquire(10, TimeUnit.SECONDS)) {
            try {
              performAdd(id, queryResults);
            } finally {
              // Don't hog the lock
              lock.release();
            }
          } else {
            count++;
            log.warn("addResults() on {} could not acquire lock after {} seconds. Attempting acquire #{}", new Object[] {id.uuid(), LOCK_SECS, count});
          }

          throw new IllegalStateException("Could not acquire lock during index() after " + count + " attempts");
        }
      } else {
        performAdd(id, queryResults);
      }
    } finally {
      sw.stop();
      id.tracer().addTiming("Cosmos:addResults", sw.elapsed(TimeUnit.MILLISECONDS));
    }
  }

  protected void performAdd(Store id, Iterable<? extends Record<?>> queryResults) throws MutationsRejectedException, TableNotFoundException, IOException {
    BatchWriter bw = null, metadataBw = null;

    try {
      // Add the values of columns to the sortableresult as we want
      Set<Index> columnsToIndex = id.columnsToIndex();

      bw = id.connector().createBatchWriter(id.dataTable(), id.writerConfig());
      metadataBw = id.connector().createBatchWriter(id.metadataTable(), id.writerConfig());

      final IndexHelper indexHelper = IndexHelper.create(columnsToIndex);
      final Text holder = new Text();
      final Set<Column> columnsAlreadyIndexed = Sets.newHashSet();

      for (Record<?> result : queryResults) {
        bw.addMutation(addDocument(id, result));
        Mutation columnMutation = new Mutation(id.uuid());
        boolean newColumnIndexed = false;

        for (Entry<Column,RecordValue<?>> entry : result.columnValues()) {
          final Column c = entry.getKey();
          final RecordValue<?> v = entry.getValue();

          if (!columnsAlreadyIndexed.contains(c)) {
            holder.set(c.name());
            columnMutation.put(PersistedStores.COLUMN_COLFAM, holder, Defaults.EMPTY_VALUE);
            columnsAlreadyIndexed.add(c);
            newColumnIndexed = true;
          }

          if (indexHelper.shouldIndex(c)) {
            for (Index index : indexHelper.indicesForColumn(c)) {
              Mutation m = getDocumentPrefix(id, result, v, index.order());

              final String direction = Order.direction(index.order());
              m.put(index.column().toString(), direction + Defaults.NULL_BYTE_STR + result.docId(), v.visibility(), Defaults.EMPTY_VALUE);

              bw.addMutation(m);
            }
          }
        }

        if (newColumnIndexed) {
          metadataBw.addMutation(columnMutation);
        }
      }
    } catch (MutationsRejectedException e) {
      log.error("Caught exception adding results for {}", id, e);
      throw e;
    } catch (TableNotFoundException e) {
      log.error("Caught exception adding results for {}", id, e);
      throw e;
    } catch (RuntimeException e) {
      log.error("Caught exception adding results for {}", id, e);
      throw e;
    } catch (IOException e) {
      log.error("Caught exception adding results for {}", id, e);
      throw e;
    } finally {
      if (null != bw) {
        bw.close();
      }
      if (null != metadataBw) {
        metadataBw.close();
      }
    }
  }

  @Override
  public void finalize(Store id) throws TableNotFoundException, MutationsRejectedException, UnexpectedStateException {
    checkNotNull(id);

    Stopwatch sw = new Stopwatch().start();

    try {
      State s = PersistedStores.getState(id);

      if (!State.LOADING.equals(s)) {
        throw unexpectedState(id, State.LOADING, s);
      }

      final State desiredState = State.LOADED;

      log.debug("Changing state for {} from {} to {}", new Object[] {id, s, desiredState});

      PersistedStores.setState(id, desiredState);
    } finally {
      sw.stop();
      id.tracer().addTiming("Cosmos:finalize", sw.elapsed(TimeUnit.MILLISECONDS));
    }
  }

  @Override
  public void index(Store id, Set<Index> columnsToIndex) throws Exception {
    checkNotNull(id);
    checkNotNull(columnsToIndex);

    Stopwatch sw = new Stopwatch().start();

    try {
      State s = PersistedStores.getState(id);

      if (!State.LOADING.equals(s) && !State.LOADED.equals(s)) {
        // stopwatch stopped by finally
        throw unexpectedState(id, new State[] {State.LOADING, State.LOADED}, s);
      }

      InterProcessMutex lock = getMutex(id);

      boolean locked = false;
      int count = 1;

      // Only perform locking when the client requests it
      if (id.lockOnUpdates()) {
        while (!locked && count < 4) {
          if (locked = lock.acquire(10, TimeUnit.SECONDS)) {
            try {
              performUpdate(id, columnsToIndex);

            } finally {
              lock.release();
            }

            return;
          } else {
            count++;
            log.warn("index() on {} could not acquire lock after {} seconds. Attempting acquire #{}", new Object[] {id.uuid(), LOCK_SECS, count});
          }

          throw new IllegalStateException("Could not acquire lock during index() after " + count + " attempts");
        }
      } else {
        performUpdate(id, columnsToIndex);
      }
    } finally {
      sw.stop();
      id.tracer().addTiming("Cosmos:index", sw.elapsed(TimeUnit.MILLISECONDS));
    }
  }

  protected void performUpdate(Store id, Set<Index> columnsToIndex) throws TableNotFoundException, UnexpectedStateException, MutationsRejectedException,
      IOException {
    final IndexHelper indexHelper = IndexHelper.create(columnsToIndex);
    final int numCols = indexHelper.columnCount();
    CloseableIterable<MultimapRecord> results = null;
    BatchWriter bw = null;

    try {
      // Add the values of columns to the sortableresult as we want future results to be indexed the same way
      id.addColumnsToIndex(columnsToIndex);

      // Get the results we have to update
      results = fetch(id);

      bw = id.connector().createBatchWriter(id.dataTable(), id.writerConfig());

      // Iterate over the results we have
      for (MultimapRecord result : results) {

        // If the cardinality of columns is greater in this result than the number of columns
        // we want to index
        if (result.columnSize() > numCols) {
          // It's more efficient to go over each column to index
          for (Column columnToIndex : indexHelper.columnIndices().keySet()) {

            // Determine if the object contains the column we need to index
            if (result.containsKey(columnToIndex)) {
              // If so, get the value(s) for that column
              final Collection<Index> indices = indexHelper.indicesForColumn(columnToIndex);
              final Collection<RecordValue<?>> values = result.get(columnToIndex);

              addIndicesForRecord(id, result, bw, indices, values);
            }
          }
        } else {
          // Otherwise it's more efficient to iterate over the columns of the result
          for (Entry<Column,RecordValue<?>> entry : result.columnValues()) {
            final Column column = entry.getKey();

            // Determine if we should index this column
            if (indexHelper.shouldIndex(column)) {
              final Collection<Index> indices = indexHelper.indicesForColumn(column);
              final Collection<RecordValue<?>> values = result.get(column);

              addIndicesForRecord(id, result, bw, indices, values);
            }
          }
        }
      }
    } finally {
      if (null != bw) {
        bw.close();
      }
      if (null != results) {
        results.close();
      }
    }
  }

  /**
   * For a QueryResult, write the Index(es) for the Column the SValues came from.
   * 
   * @param id
   * @param result
   * @param bw
   * @param indices
   * @param values
   * @throws MutationsRejectedException
   * @throws IOException
   */
  protected void addIndicesForRecord(Store id, MultimapRecord result, BatchWriter bw, Collection<Index> indices, Collection<RecordValue<?>> values)
      throws MutationsRejectedException, IOException {
    // Place an Index entry for each value in each direction defined
    for (Index index : indices) {
      for (RecordValue<?> value : values) {
        // TODO Defer to the implementation of the RecordValue to get lexicographically sortedness
        Mutation m = getDocumentPrefix(id, result, value, index.order());

        final String direction = Order.direction(index.order());
        m.put(index.column().toString(), direction + Defaults.NULL_BYTE_STR + result.docId(), value.visibility(), Defaults.EMPTY_VALUE);
        bw.addMutation(m);
      }
    }
  }

  @Override
  public CloseableIterable<Column> columns(Store id) throws TableNotFoundException, UnexpectedStateException {
    checkNotNull(id);

    final String description = "Cosmos:columns";
    Stopwatch sw = new Stopwatch().start();

    try {
      State s = PersistedStores.getState(id);

      if (!State.LOADING.equals(s) && !State.LOADED.equals(s)) {
        // Stopwatch stopped by finally
        throw unexpectedState(id, new State[] {State.LOADING, State.LOADED}, s);
      }

      return PersistedStores.columns(id, description, sw);
    } catch (TableNotFoundException e) {
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    } catch (UnexpectedStateException e) {
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    }
  }

  @Override
  public CloseableIterable<MultimapRecord> fetch(Store id) throws TableNotFoundException, UnexpectedStateException {
    checkNotNull(id);

    final String description = "Cosmos:fetch";
    Stopwatch sw = new Stopwatch().start();

    try {
      State s = PersistedStores.getState(id);

      if (!State.LOADING.equals(s) && !State.LOADED.equals(s)) {
        throw unexpectedState(id, new State[] {State.LOADING, State.LOADED}, s);
      }

      BatchScanner bs = id.connector().createBatchScanner(id.dataTable(), id.auths(), id.readThreads());
      bs.setRanges(Collections.singleton(Range.prefix(id.uuid())));
      bs.fetchColumnFamily(Defaults.DOCID_FIELD_NAME_TEXT);

      // Handles stoping the stopwatch
      return CloseableIterable.transform(bs, new IndexToMultimapRecord(this, id), id.tracer(), description, sw);
    } catch (TableNotFoundException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    } catch (UnexpectedStateException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    } catch (RuntimeException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    }
    // no finally as the trace is stopped by the CloseableIterable
  }

  @Override
  public PagedResults<MultimapRecord> fetch(Store id, Paging limits) throws TableNotFoundException, UnexpectedStateException {
    checkNotNull(id);
    checkNotNull(limits);

    CloseableIterable<MultimapRecord> results = fetch(id);

    return new PagedResults<MultimapRecord>(results, limits);
  }

  @Override
  public CloseableIterable<MultimapRecord> fetch(Store id, Column column, String value) throws TableNotFoundException, UnexpectedStateException {
    checkNotNull(id);
    checkNotNull(column);
    checkNotNull(value);

    final String description = "Cosmos:fetchWithColumnValue";
    Stopwatch sw = new Stopwatch().start();

    try {
      State s = PersistedStores.getState(id);

      if (!State.LOADING.equals(s) && !State.LOADED.equals(s)) {
        throw unexpectedState(id, new State[] {State.LOADING, State.LOADED}, s);
      }

      BatchScanner bs = id.connector().createBatchScanner(id.dataTable(), id.auths(), id.readThreads());
      bs.setRanges(Collections.singleton(Range.exact(id.uuid() + Defaults.NULL_BYTE_STR + value)));
      bs.fetchColumnFamily(new Text(column.name()));

      // Filter on cq-prefix to only look at the ordering we want
      IteratorSetting filter = new IteratorSetting(50, "cqFilter", OrderFilter.class);
      filter.addOption(OrderFilter.PREFIX, Order.direction(Order.ASCENDING));
      bs.addScanIterator(filter);

      return CloseableIterable.transform(bs, new IndexToMultimapRecord(this, id), id.tracer(), description, sw);
    } catch (TableNotFoundException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    } catch (UnexpectedStateException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    } catch (RuntimeException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    }
    // no finally as the trace is stopped by the CloseableIterable
  }

  @Override
  public PagedResults<MultimapRecord> fetch(Store id, Column column, String value, Paging limits) throws TableNotFoundException, UnexpectedStateException {
    checkNotNull(limits);

    CloseableIterable<MultimapRecord> results = fetch(id, column, value);

    return PagedResults.create(results, limits);
  }

  @Override
  public CloseableIterable<MultimapRecord> fetch(Store id, Index ordering) throws TableNotFoundException, UnexpectedStateException, UnindexedColumnException {
    return fetch(id, ordering, true);
  }

  @Override
  public CloseableIterable<MultimapRecord> fetch(Store id, Index ordering, boolean duplicateUidsAllowed) throws TableNotFoundException,
      UnexpectedStateException, UnindexedColumnException {
    checkNotNull(id);
    checkNotNull(ordering);

    final String description = "Cosmos:fetchWithIndex";
    Stopwatch sw = new Stopwatch().start();

    try {
      State s = PersistedStores.getState(id);

      if (!State.LOADING.equals(s) && !State.LOADED.equals(s)) {
        throw unexpectedState(id, new State[] {State.LOADING, State.LOADED}, s);
      }

      Index.define(ordering.column());

      if (!id.columnsToIndex().contains(ordering)) {
        log.error("{} is not indexed by {}", ordering, id);

        throw new UnindexedColumnException();
      }

      Scanner scanner = id.connector().createScanner(id.dataTable(), id.auths());
      scanner.setRange(Range.prefix(id.uuid()));
      scanner.fetchColumnFamily(new Text(ordering.column().name()));
      scanner.setBatchSize(200);

      // Filter on cq-prefix to only look at the ordering we want
      IteratorSetting filter = new IteratorSetting(50, "cqFilter", OrderFilter.class);
      filter.addOption(OrderFilter.PREFIX, Order.direction(ordering.order()));
      scanner.addScanIterator(filter);

      // If the client has told us they don't want duplicate records, lets not give them duplicate records
      if (duplicateUidsAllowed) {
        return CloseableIterable.transform(scanner, new IndexToMultimapRecord(this, id), id.tracer(), description, sw);
      } else {
        return CloseableIterable.filterAndTransform(scanner, new DedupingPredicate(), new IndexToMultimapRecord(this, id), id.tracer(), description, sw);
      }
    } catch (TableNotFoundException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    } catch (UnexpectedStateException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    } catch (RuntimeException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    }
    // no finally as the trace is stopped by the CloseableIterable
  }

  @Override
  public PagedResults<MultimapRecord> fetch(Store id, Index ordering, Paging limits) throws TableNotFoundException, UnexpectedStateException,
      UnindexedColumnException {
    checkNotNull(id);
    checkNotNull(limits);

    CloseableIterable<MultimapRecord> results = fetch(id, ordering);

    return PagedResults.create(results, limits);
  }

  @Override
  public CloseableIterable<Entry<RecordValue<?>,Long>> groupResults(Store id, Column column) throws TableNotFoundException, UnexpectedStateException,
      UnindexedColumnException {
    checkNotNull(id);

    Stopwatch sw = new Stopwatch().start();
    final String description = "Cosmos:groupResults";

    try {
      State s = PersistedStores.getState(id);

      if (!State.LOADING.equals(s) && !State.LOADED.equals(s)) {
        sw.stop();
        throw unexpectedState(id, new State[] {State.LOADING, State.LOADED}, s);
      }

      checkNotNull(column);

      Text colf = new Text(column.name());

      BatchScanner bs = id.connector().createBatchScanner(id.dataTable(), id.auths(), id.readThreads());
      bs.setRanges(Collections.singleton(Range.prefix(id.uuid())));
      bs.fetchColumnFamily(colf);

      // Filter on cq-prefix to only look at the ordering we want
      IteratorSetting filter = new IteratorSetting(50, "cqFilter", OrderFilter.class);
      filter.addOption(OrderFilter.PREFIX, Order.FORWARD);
      bs.addScanIterator(filter);

      IteratorSetting cfg = new IteratorSetting(60, GroupByRowSuffixIterator.class);
      bs.addScanIterator(cfg);

      return CloseableIterable.transform(bs, new GroupByFunction(), id.tracer(), description, sw);
    } catch (TableNotFoundException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    } catch (UnexpectedStateException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    } catch (RuntimeException e) {
      // In the exceptional case, stop the timer
      sw.stop();
      id.tracer().addTiming(description, sw.elapsed(TimeUnit.MILLISECONDS));
      throw e;
    }
    // no finally as the trace is stopped by the CloseableIterable
  }

  @Override
  public PagedResults<Entry<RecordValue<?>,Long>> groupResults(Store id, Column column, Paging limits) throws TableNotFoundException, UnexpectedStateException,
      UnindexedColumnException {
    checkNotNull(limits);

    CloseableIterable<Entry<RecordValue<?>,Long>> results = groupResults(id, column);

    return PagedResults.create(results, limits);
  }

  @Override
  public MultimapRecord contents(Store id, String docId) throws TableNotFoundException, UnexpectedStateException {
    checkNotNull(id);
    checkNotNull(docId);

    // Omit tracing here just due to sheer magnitude of these calls.

    State s = PersistedStores.getState(id);

    if (!State.LOADING.equals(s) && !State.LOADED.equals(s)) {
      throw unexpectedState(id, new State[] {State.LOADING, State.LOADED}, s);
    }

    Scanner scanner = id.connector().createScanner(id.dataTable(), id.auths());
    scanner.setRange(Range.exact(id.uuid() + Defaults.NULL_BYTE_STR + docId));
    scanner.fetchColumnFamily(Defaults.CONTENTS_COLFAM_TEXT);

    Iterator<Entry<Key,Value>> iter = scanner.iterator();
    if (!iter.hasNext()) {
      scanner.close();

      throw new NoSuchElementException("No such result for " + docId + " in " + id.uuid());
    } else {
      Value value = iter.next().getValue();
      scanner.close();

      return KeyValueToMultimapQueryResult.transform(value);
    }
  }

  @Override
  public void delete(Store id) throws TableNotFoundException, MutationsRejectedException, UnexpectedStateException {
    checkNotNull(id);

    Stopwatch sw = new Stopwatch().start();

    try {
      State s = PersistedStores.getState(id);

      if (!State.LOADING.equals(s) && !State.LOADED.equals(s)) {
        throw unexpectedState(id, new State[] {State.LOADING, State.LOADED}, s);
      }

      final State desiredState = State.DELETING;

      log.debug("Changing state for {} from {} to {}", new Object[] {id, s, desiredState});

      PersistedStores.setState(id, desiredState);

      // Delete of the Keys
      BatchDeleter bd = null;
      try {
        bd = id.connector().createBatchDeleter(id.dataTable(), id.auths(), id.readThreads(), id.writerConfig());
        bd.setRanges(Collections.singleton(Range.prefix(id.uuid())));

        bd.delete();
      } finally {
        if (null != bd) {
          bd.close();
        }
      }

      log.debug("Removing state for {}", id);

      PersistedStores.remove(id);
    } finally {
      sw.stop();
      id.tracer().addTiming("Cosmos:delete", sw.elapsed(TimeUnit.MILLISECONDS));

      // Be nice and when the client deletes these results, automatically flush the traces for them too
      id.sendTraces();
    }
  }

  protected Mutation getDocumentPrefix(Store id, Record<?> record, RecordValue<?> value, Order order) {
    byte[] b = id.uuid().getBytes();
    if (Order.ASCENDING.equals(order)) {
      b = value.lexicographicValue();
    } else {
      b = value.reverseLexicographicValue();
    }

    return getDocumentPrefix(id, record, b);
  }

  protected Mutation getDocumentPrefix(Store id, Record<?> record, byte[] suffix) {
    final Text t = new Text();
    byte[] b = id.uuid().getBytes();
    t.append(b, 0, b.length);
    t.append(new byte[] {0}, 0, 1);
    t.append(suffix, 0, suffix.length);

    return new Mutation(t);
  }

  protected Mutation addDocument(Store id, Record<?> queryResult) throws IOException {
    Mutation m = getDocumentPrefix(id, queryResult, queryResult.docId().getBytes(Constants.UTF8));

    // Store the docId as a searchable entry
    m.put(Defaults.DOCID_FIELD_NAME, Order.FORWARD + Defaults.NULL_BYTE_STR + queryResult.docId(), queryResult.documentVisibility(), Defaults.EMPTY_VALUE);

    // Write the contents for this record once
    m.put(Defaults.CONTENTS_COLFAM_TEXT, new Text(), queryResult.documentVisibility(), queryResult.toValue());

    return m;
  }

  protected UnexpectedStateException unexpectedState(Store id, State[] expected, State actual) {
    return new UnexpectedStateException("Invalid state " + id + " for " + id + ". Expected one of " + Arrays.asList(expected) + " but was " + actual);
  }

  protected UnexpectedStateException unexpectedState(Store id, State expected, State actual) {
    return new UnexpectedStateException("Invalid state " + id + " for " + id + ". Expected " + expected + " but was " + actual);
  }

  protected final InterProcessMutex getMutex(Store id) {
    return new InterProcessMutex(curator, Defaults.CURATOR_PREFIX + id.uuid());
  }

}