Log.java

package com.leansoft.luxun.log;

import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.leansoft.bigqueue.FanOutQueueImplEx;
import com.leansoft.bigqueue.FanOutQueueImplEx.BatchReadResult;
import com.leansoft.bigqueue.IFanOutQueueEx;
import com.leansoft.luxun.common.annotations.NotThreadSafe;
import com.leansoft.luxun.common.exception.MessageSizeTooLargeException;
import com.leansoft.luxun.mx.BrokerTopicStat;
import com.leansoft.luxun.mx.LogFlushStats;
import com.leansoft.luxun.mx.LogStats;
import com.leansoft.luxun.utils.Closer;
import com.leansoft.luxun.utils.Utils;


/**
 * a log is a message set backed by a persistent fanout queue
 *
 */
public class Log implements ILog {
	
	private final Logger logger = LoggerFactory.getLogger(Log.class);
	
	public final File baseDir;
	
	public final String topic;
	
	final int maxItemCountBeforeFlush;
	
	final boolean needRecovery;
	
    private final AtomicBoolean flushLock = new AtomicBoolean(false);
	
	private final AtomicInteger unflushed = new AtomicInteger(0);
	
    private final AtomicLong lastflushedTime = new AtomicLong(0L);
	
    // a log is internally backed by an append only fanout queue
	private IFanOutQueueEx foQueue;
	
    private final LogStats logStats = new LogStats(this);
	
	private final int maxMessageSize;
	
	public final File realLogDir;
	
	public Log(File baseDir, String topic, 
			int maxItemCountBeforeFlush,
			boolean needRecovery,
			int pageSize,
			int maxMessageSize) throws IOException {
		this.baseDir = baseDir;
		this.maxItemCountBeforeFlush = maxItemCountBeforeFlush;
		this.needRecovery = needRecovery;
		this.maxMessageSize = maxMessageSize;
		this.topic = topic;

		String logDir = this.baseDir.getAbsolutePath();
		if (!logDir.endsWith(File.separator)) {
			logDir += File.separator;
		}
		logDir +=  topic;
		realLogDir = new File(logDir);
		
		this.logStats.setMbeanName("luxun:type=luxun.logs." + topic);
		Utils.registerMBean(logStats);
		
		this.foQueue = new FanOutQueueImplEx(baseDir.getAbsolutePath(), topic, pageSize);
	}
	
    /**
     * delete all log page files in this topic <br/>
     * The log directory will be removed also.
     * @throws IOException 
     */
	public void delete() {
		this.close();
		Utils.deleteDirectory(realLogDir);
		logger.info("Deleted " + this);
	}
	
	
	@Override
	public void close() {
		Closer.closeQuietly(this.foQueue, logger);
		this.foQueue = null;
		Utils.unregisterMBean(this.logStats);
	}
	
    public long getLastFlushedTime() {
        return lastflushedTime.get();
    }
    
    @Override
    public String toString() {
        return "Log [dir=" + this.realLogDir + ", lastflushedTime=" + //
        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(new Date(lastflushedTime.get())) + "]";
    }

	@Override
	public byte[] read(long index) throws IOException {
		return this.foQueue.get(index);
	}

	@Override
	public long append(byte[] item) throws IOException {
		// validate message size
		if (item.length > maxMessageSize) {
			throw new MessageSizeTooLargeException("payload size of " + item.length + " larger than " + maxMessageSize);
		}
		
		long index = this.foQueue.enqueue(item);
		
		unflushed.incrementAndGet();
		if (maxItemCountBeforeFlush > 0) {
			this.maybeFlush();
		}
		
		BrokerTopicStat.getBrokerTopicStat(topic).recordMessagesIn(1);
		BrokerTopicStat.getBrokerAllTopicStat().recordMessagesIn(1);
		this.logStats.incrementAppendedMessageCount();
		
		return index;
	}

	/**
	 * total number of messages in this topic(some old messages may have been deleted)
	 */
	@Override
	public long getSize() {
		return this.foQueue.size();
	}

	@Override
	public long getClosestIndex(long timestamp) throws IOException {
		return this.foQueue.findClosestIndex(timestamp);
	}

	@Override
	public void removeBefore(long timestamp) throws IOException {
		this.foQueue.removeBefore(timestamp);
	}
	
	private void maybeFlush() {
		if (unflushed.get() >= maxItemCountBeforeFlush) {
			flush();
		}
	}

	@Override
	public void flush() {
		if (unflushed.get() == 0) return; // nothing to flush
		
		if (flushLock.compareAndSet(false, true)) {
			try {
				long startTime = System.currentTimeMillis();
				this.foQueue.flush();
				long elapsedTime = System.currentTimeMillis() - startTime;
				LogFlushStats.recordFlushRequest(elapsedTime);
				unflushed.set(0);
				lastflushedTime.set(System.currentTimeMillis());
			} finally {
				flushLock.set(false);
			}
		}
	}

	@Override
	public void limitBackFileSize(long sizeLimit) throws IOException {
		this.foQueue.limitBackFileSize(sizeLimit);
	}

	@Override
	public boolean isEmpty() {
		return this.foQueue.isEmpty();
	}
	
	@Override
	public long getFrontIndex() {
		return this.foQueue.getFrontIndex();
	}

	@Override
	public long getRearIndex() {
		return this.foQueue.getRearIndex();
	}

	@Override
	public long getBackFileSize() throws IOException {
		return this.foQueue.getBackFileSize();
	}

	@Override
	public int getItemLength(long index) throws IOException {
		return this.foQueue.getLength(index);
	}

	@Override
	public long getTimestamp(long index) throws IOException {
		return this.foQueue.getTimestamp(index);
	}

	@Override
	public byte[] read(String fanoutId) throws IOException {
		return this.foQueue.dequeue(fanoutId);
	}

	@Override
	public int getItemLength(String fanoutId) throws IOException {
		return this.foQueue.peekLength(fanoutId);
	}

	@Override
	public long getFrontIndex(String fanoutId) throws IOException {
		return this.foQueue.getFrontIndex(fanoutId);
	}

	@Override
	public long getSize(String fanoutId) throws IOException {
		return this.foQueue.size(fanoutId);
	}

	@Override
	public boolean isEmpty(String fanoutId) throws IOException {
		return this.foQueue.isEmpty(fanoutId);
	}

	@Override
	public Lock getQueueFrontWriteLock(String fanoutId) throws IOException {
		return this.foQueue.getQueueFrontWriteLock(fanoutId);
	}

	@Override
	public Lock getInnerArrayReadLock() {
		return this.foQueue.getInnerArrayReadLock();
	}

	@Override
	public int getNumberOfBackFiles() {
		return this.foQueue.getNumberOfBackFiles();
	}

	@Override
	@NotThreadSafe
	public BatchReadResult batchRead(String fanoutId, int maxFetchSize)
			throws IOException {
		return this.foQueue.batchDequeue(fanoutId, maxFetchSize);
	}
}