Hourglass.java

package proj.zoie.hourglass.impl;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import javax.management.NotCompliantMBeanException;
import javax.management.StandardMBean;

import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.store.Directory;

import proj.zoie.api.DefaultDirectoryManager;
import proj.zoie.api.DirectoryManager;
import proj.zoie.api.DocIDMapper;
import proj.zoie.api.Zoie;
import proj.zoie.api.ZoieException;
import proj.zoie.api.ZoieMultiReader;
import proj.zoie.api.indexing.IndexReaderDecorator;
import proj.zoie.api.indexing.ZoieIndexableInterpreter;
import proj.zoie.hourglass.mbean.HourglassAdmin;
import proj.zoie.hourglass.mbean.HourglassAdminMBean;
import proj.zoie.impl.indexing.ZoieConfig;
import proj.zoie.impl.indexing.ZoieSystem;

public class Hourglass<R extends IndexReader, D> implements Zoie<R, D> {
  public static final Logger log = Logger.getLogger(Hourglass.class);
  private final HourglassDirectoryManagerFactory _dirMgrFactory;
  private final ZoieIndexableInterpreter<D> _interpreter;
  private final IndexReaderDecorator<R> _decorator;
  private final ZoieConfig _zConfig;
  private volatile ZoieSystem<R, D> _currentZoie;
  private volatile boolean _isShutdown = false;
  final ReentrantReadWriteLock _shutdownLock = new ReentrantReadWriteLock();
  private final ReentrantLock _consumeLock = new ReentrantLock();
  private final HourglassReaderManager<R, D> _readerMgr;
  private volatile String _currentVersion = null;
  private long _freshness = 1000;
  final HourGlassScheduler _scheduler;
  public volatile long SLA = 4; // getIndexReaders should return in 4ms or a warning is logged
  @SuppressWarnings("rawtypes")
  private List<HourglassListener> _hourglassListeners;

  @SuppressWarnings({ "rawtypes" })
  public Hourglass(HourglassDirectoryManagerFactory dirMgrFactory,
      ZoieIndexableInterpreter<D> interpreter, IndexReaderDecorator<R> readerDecorator,
      ZoieConfig zoieConfig, List<HourglassListener> hourglassListeners) {
    _zConfig = zoieConfig;
    _dirMgrFactory = dirMgrFactory;
    if (hourglassListeners == null) {
      hourglassListeners = Collections.emptyList();
    } else {
      hourglassListeners = new CopyOnWriteArrayList<HourglassListener>(hourglassListeners);
    }
    _hourglassListeners = hourglassListeners;
    _scheduler = _dirMgrFactory.getScheduler();
    _dirMgrFactory.clearRecentlyChanged();
    _interpreter = interpreter;
    _decorator = readerDecorator;
    List<ZoieMultiReader<R>> archives;
    List<ZoieSystem<R, D>> archiveZoies;
    if (_dirMgrFactory.getScheduler().isAppendOnly()) {
      archives = loadArchives();
      archiveZoies = Collections.emptyList();
    } else {
      archives = Collections.emptyList();
      archiveZoies = loadArchiveZoies();
    }
    _readerMgr = new HourglassReaderManager<R, D>(this, _dirMgrFactory, _decorator, archives,
        archiveZoies, hourglassListeners);
    _currentVersion = _dirMgrFactory.getArchivedVersion();
    _currentZoie = _readerMgr.retireAndNew(null);
    _currentZoie.start();
    _freshness = zoieConfig.getFreshness();
    log.info("start Hourglass at version: " + _currentVersion);
  }

  @SuppressWarnings("rawtypes")
  public Hourglass(HourglassDirectoryManagerFactory dirMgrFactory,
      ZoieIndexableInterpreter<D> interpreter, IndexReaderDecorator<R> readerDecorator,
      ZoieConfig zoieConfig) {
    this(dirMgrFactory, interpreter, readerDecorator, zoieConfig, Collections
        .<HourglassListener> emptyList());
  }

  @SuppressWarnings({ "rawtypes" })
  public Hourglass(HourglassDirectoryManagerFactory dirMgrFactory,
      ZoieIndexableInterpreter<D> interpreter, IndexReaderDecorator<R> readerDecorator,
      ZoieConfig zoieConfig, HourglassListener hourglassListener) {
    this(dirMgrFactory, interpreter, readerDecorator, zoieConfig, Arrays.asList(hourglassListener));
  }

  protected List<ZoieSystem<R, D>> loadArchiveZoies() {
    List<ZoieSystem<R, D>> archives = new ArrayList<ZoieSystem<R, D>>();
    long t0 = System.currentTimeMillis();
    List<File> dirs = _dirMgrFactory.getAllArchivedDirs();
    for (File dir : dirs) {
      try {
        DirectoryManager dirMgr = new DefaultDirectoryManager(dir, _dirMgrFactory.getMode());
        ZoieSystem<R, D> zoie = new ZoieSystem<R, D>(dirMgr, _interpreter, _decorator, _zConfig);
        zoie.start();
        archives.add(zoie);
      } catch (Exception e) {
        log.error("Load index: " + dir + " failed.", e);
      }
    }
    log.info("load " + dirs.size() + " archived indices of " + getSizeBytes() + " bytes in "
        + (System.currentTimeMillis() - t0) + "ms");
    return archives;
  }

  protected List<ZoieMultiReader<R>> loadArchives() {
    List<ZoieMultiReader<R>> archives = new ArrayList<ZoieMultiReader<R>>();
    long t0 = System.currentTimeMillis();
    List<Directory> dirs = _dirMgrFactory.getAllArchivedDirectories();
    for (Directory dir : dirs) {
      DirectoryReader reader;
      try {
        reader = DirectoryReader.open(dir);
        ZoieMultiReader<R> zoiereader = new ZoieMultiReader<R>(reader, _decorator);

        // Initialize docIdMapper
        DocIDMapper mapper = _zConfig.getDocidMapperFactory().getDocIDMapper(zoiereader);
        zoiereader.setDocIDMapper(mapper);

        archives.add(zoiereader);
      } catch (CorruptIndexException e) {
        log.error("corruptedIndex", e);
      } catch (IOException e) {
        log.error("IOException", e);
      }
    }
    log.info("load " + dirs.size() + " archived indices of " + getSizeBytes() + " bytes in "
        + (System.currentTimeMillis() - t0) + "ms");
    return archives;
  }

  ZoieSystem<R, D> createZoie(DirectoryManager dirmgr) {
    return new ZoieSystem<R, D>(dirmgr, _interpreter, _decorator, _zConfig);
  }

  public ZoieConfig getzConfig() {
    return _zConfig;
  }

  public ZoieSystem<R, D> getCurrentZoie() {
    return _currentZoie;
  }

  public HourglassDirectoryManagerFactory getDirMgrFactory() {
    return _dirMgrFactory;
  }

  /*
   * (non-Javadoc)
   * @see proj.zoie.api.IndexReaderFactory#getAnalyzer()
   */
  @Override
  public Analyzer getAnalyzer() {
    return _zConfig.getAnalyzer();
  }

  /**
   * return a list of ZoieMultiReaders. These readers are reference counted and this method
   * should be used in pair with returnIndexReaders(List<ZoieMultiReader<R>> readers) {@link #returnIndexReaders(List)}.
   * It is typical that we create a MultiReader from these readers. When creating MultiReader, it should be created with
   * the closeSubReaders parameter set to false in order to do reference counting correctly.
   * <br> If this indexing system is already shut down, then we return an empty list.
   * @see proj.zoie.hourglass.impl.Hourglass#returnIndexReaders(List)
   * @see proj.zoie.api.IndexReaderFactory#getIndexReaders()
   */
  @Override
  public List<ZoieMultiReader<R>> getIndexReaders() throws IOException {
    long t0 = System.currentTimeMillis();
    try {
      _shutdownLock.readLock().lock();
      if (_isShutdown) {
        log.warn("System already shut down. No search request allowed.");
        List<ZoieMultiReader<R>> list = new ArrayList<ZoieMultiReader<R>>();
        return list;// if already shutdown, return an empty list
      }
      try {
        cacheLock.lock();
        if (System.currentTimeMillis() - lastupdate > _freshness) {
          updateCachedReaders();
        }
        List<ZoieMultiReader<R>> rlist = list;
        for (ZoieMultiReader<R> r : rlist) {
          r.incZoieRef();
        }
        t0 = System.currentTimeMillis() - t0;
        if (t0 > SLA) {
          log.warn("getIndexReaders returned in " + t0 + "ms more than " + SLA + "ms");
        }
        return rlist;
      } finally {
        cacheLock.unlock();
      }
    } finally {
      _shutdownLock.readLock().unlock();
    }
  }

  /**
   * not thread safe. should be properly lock. Right now we have two places to use it
   * and locked by the shutdown lock. If it gets more complicated, we should use separate
   * lock.
   * @throws IOException
   */
  private void updateCachedReaders() throws IOException {
    if (log.isDebugEnabled()) {
      log.debug("updating reader cache");
    }
    List<ZoieMultiReader<R>> olist = list;
    returnIndexReaders(olist);
    if (log.isDebugEnabled()) {
      log.debug("getting new reader from reader cache");
    }
    list = _readerMgr.getIndexReaders();
    if (log.isDebugEnabled()) {
      log.debug("reader updated with size: " + list.size());
    }
    lastupdate = System.currentTimeMillis();
  }

  /**
   * not thread safe. should be properly lock. Right now we have two places to use it
   * and locked by the shutdown lock. If it gets more complicated, we should use separate
   * lock.
   */
  private void clearCachedReaders() {
    List<ZoieMultiReader<R>> olist = list;
    returnIndexReaders(olist);
    list = null;
    lastupdate = 0;
  }

  private volatile long lastupdate = 0;
  private volatile List<ZoieMultiReader<R>> list = new ArrayList<ZoieMultiReader<R>>();
  private final ReentrantLock cacheLock = new ReentrantLock();

  /*
   * (non-Javadoc)
   * @see proj.zoie.api.IndexReaderFactory#returnIndexReaders(java.util.List)
   */
  @Override
  public void returnIndexReaders(List<ZoieMultiReader<R>> readers) {
    long t0 = System.currentTimeMillis();
    _currentZoie.returnIndexReaders(readers);
    t0 = System.currentTimeMillis() - t0;
    if (t0 > SLA) {
      log.warn("returnIndexReaders returned in " + t0 + "ms more than " + SLA + "ms");
    }
  }

  private void clearFromArchives(Collection<DataEvent<D>> data) throws ZoieException {
    if (_dirMgrFactory.getScheduler().isAppendOnly()) return;

    if (data != null && data.size() > 0) {
      List<DataEvent<D>> deletes = new ArrayList<DataEvent<D>>(data.size());
      for (DataEvent<D> event : data) {
        deletes.add(new DataEvent<D>(event.getData(), event.getVersion(), true));
      }
      for (ZoieSystem<R, D> zoie : _readerMgr.getArchiveZoies()) {
        zoie.consume(deletes);
      }
    }
  }

  /*
   * (non-Javadoc)
   * @see proj.zoie.api.DataConsumer#consume(java.util.Collection)
   */
  @Override
  public void consume(Collection<DataEvent<D>> data) throws ZoieException {
    try {
      _consumeLock.lock(); // one at a time so we don't mess up during forward rolling.
                           // also we cannot do two consumptions at the same time anyway.
      try {
        _shutdownLock.readLock().lock();
        if (data == null || data.size() == 0) return;
        if (_isShutdown) {
          log.warn("System already shut down. Rejects indexing request.");
          return; // if the system is already shut down, we don't do anything.
        }
        // need to check time boundary. When we hit boundary, we need to trigger DM to
        // use new dir for zoie and the old one will be archive.
        if (!_dirMgrFactory.updateDirectoryManager()) {
          clearFromArchives(data);
          _currentZoie.consume(data);
        } else {
          // new time period
          _currentZoie = _readerMgr.retireAndNew(_currentZoie);
          _currentZoie.start();
          clearFromArchives(data);
          _currentZoie.consume(data);
        }
      } finally {
        _shutdownLock.readLock().unlock();
      }
    } finally {
      _consumeLock.unlock();
    }
  }

  @Override
  public void shutdown() {
    try {
      _shutdownLock.writeLock().lock();
      if (_isShutdown) {
        log.info("system already shut down");
        return;
      }
      _isShutdown = true;
    } finally {
      _shutdownLock.writeLock().unlock();
    }
    clearCachedReaders();
    _readerMgr.shutdown();

    log.info("shut down complete.");
  }

  /*
   * (non-Javadoc)
   * @see proj.zoie.api.DataConsumer#getVersion()
   */
  @Override
  public String getVersion() {
    // _currentVersion = Math.max(_currentVersion, _currentZoie.getCurrentVersion());
    if (_currentZoie.getCurrentVersion() != null) {
      if (_currentVersion == null) {
        return _currentVersion = _currentZoie.getCurrentVersion();
      } else {
        _currentVersion = _zConfig.getVersionComparator().compare(_currentZoie.getCurrentVersion(),
          _currentVersion) < 0 ? _currentVersion : _currentZoie.getCurrentVersion();
      }
    }

    return _currentVersion;
  }

  /*
   * (non-Javadoc)
   * @see proj.zoie.api.DataConsumer#getVersionComparator()
   */
  @Override
  public Comparator<String> getVersionComparator() {
    return _zConfig.getVersionComparator();
  }

  public long getSizeBytes() {
    return _dirMgrFactory.getDiskIndexSizeBytes();
  }

  @Override
  public void syncWithVersion(long timeInMillis, String version) throws ZoieException {
    if (_currentZoie != null) {
      _currentZoie.syncWithVersion(timeInMillis, version);
    }
  }

  @Override
  public StandardMBean getStandardMBean(String name) {
    if (name.equals(HOURGLASSADMIN)) {
      try {
        return new StandardMBean(getAdminMBean(), HourglassAdminMBean.class);
      } catch (NotCompliantMBeanException e) {
        log.info(e);
        return null;
      }
    }
    return null;
  }

  @Override
  public HourglassAdminMBean getAdminMBean() {
    return new HourglassAdmin(this);
  }

  public static String HOURGLASSADMIN = "hourglass-admin";

  @Override
  public String[] getStandardMBeanNames() {
    return new String[] { HOURGLASSADMIN };
  }

  @Override
  public void start() {
    log.info("starting Hourglass... already done due by auto-start");
  }

  @Override
  public void flushEvents(long timeout) throws ZoieException {
    _currentZoie.flushEvents(timeout);
  }

  @Override
  public String getCurrentReaderVersion() {
    return _currentZoie == null ? null : _currentZoie.getCurrentReaderVersion();
  }

  public void addHourglassListener(HourglassListener<R, D> hourglassListener) {
    if (hourglassListener != null) {
      _hourglassListeners.add(hourglassListener);
    }
  }
}