/**
* $Id: MapedFileQueue.java 1831 2013-05-16 01:39:51Z shijia.wxr $
*/
package com.alibaba.rocketmq.store;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.rocketmq.common.MixAll;
import com.alibaba.rocketmq.common.UtilALl;
/**
* �洢���У����ݶ�ʱɾ������������<br>
* �������ɶ���ļ����
*
* @author vintage.wang@gmail.com shijia.wxr@taobao.com
*/
public class MapedFileQueue {
private static final Logger log = LoggerFactory.getLogger(MixAll.StoreLoggerName);
// ÿ�δ���ɾ���ļ������ɾ�����ٸ��ļ�
private static final int DeleteFilesBatchMax = 30;
// �ļ��洢λ��
private final String storePath;
// ÿ���ļ��Ĵ�С
private final int mapedFileSize;
// ˢ��ˢ������
private long committedWhere = 0;
// �����ļ�
private final List<MapedFile> mapedFiles = new ArrayList<MapedFile>();
// ��������mapedFiles��
private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
// Ԥ����MapedFile�������
private final AllocateMapedFileService allocateMapedFileService;
// ���һ����Ϣ�洢ʱ��
private volatile long storeTimestamp = 0;
public MapedFileQueue(final String storePath, int mapedFileSize,
AllocateMapedFileService allocateMapedFileService) {
this.storePath = storePath;
this.mapedFileSize = mapedFileSize;
this.allocateMapedFileService = allocateMapedFileService;
}
public MapedFile getMapedFileByTime(final long timestamp) {
Object[] mfs = this.copyMapedFiles(0);
if (null == mfs)
return null;
for (int i = 0; i < mfs.length; i++) {
MapedFile mapedFile = (MapedFile) mfs[i];
if (mapedFile.getLastModifiedTimestamp() >= timestamp) {
return mapedFile;
}
}
return (MapedFile) mfs[mfs.length - 1];
}
/**
* recoverʱ���ã�����Ҫ����
*/
public void truncateDirtyFiles(long offset) {
List<MapedFile> willRemoveFiles = new ArrayList<MapedFile>();
for (MapedFile file : this.mapedFiles) {
long fileTailOffset = file.getFileFromOffset() + this.mapedFileSize;
if (fileTailOffset > offset) {
if (offset >= file.getFileFromOffset()) {
file.setWrotePostion((int) (offset % this.mapedFileSize));
file.setCommittedPosition((int) (offset % this.mapedFileSize));
}
else {
// ���ļ�ɾ����
file.destroy(1000);
willRemoveFiles.add(file);
}
}
}
this.deleteExpiredFile(willRemoveFiles);
}
public boolean load() {
File dir = new File(this.storePath);
File[] files = dir.listFiles();
if (files != null) {
// ascending order
Arrays.sort(files);
for (File file : files) {
// У���ļ���С�Ƿ�ƥ��
if (file.length() != this.mapedFileSize) {
log.warn(file + "\t" + file.length()
+ " length not matched message store config value, ignore it");
return true;
}
// �ָ�����
try {
MapedFile mapedFile = new MapedFile(file.getPath(), mapedFileSize);
mapedFile.setWrotePostion(this.mapedFileSize);
mapedFile.setCommittedPosition(this.mapedFileSize);
this.mapedFiles.add(mapedFile);
log.info("load " + file.getPath() + " OK");
}
catch (IOException e) {
log.error("load file " + file + " error", e);
return false;
}
}
}
return true;
}
/**
* ˢ�̽�������˶���
*/
public long howMuchFallBehind() {
if (this.mapedFiles.isEmpty())
return 0;
long committed = this.committedWhere;
if (committed != 0) {
MapedFile mapedFile = this.getLastMapedFile();
if (mapedFile != null) {
return (mapedFile.getFileFromOffset() + mapedFile.getWrotePostion()) - committed;
}
}
return 0;
}
/**
* ��ȡ���е���СOffset���������Ϊ�գ���-1
*/
public long getMinOffset() {
try {
this.readWriteLock.readLock().lock();
if (!this.mapedFiles.isEmpty()) {
return this.mapedFiles.get(0).getFileFromOffset();
}
}
catch (Exception e) {
log.error("getMinOffset has exception.", e);
}
finally {
this.readWriteLock.readLock().unlock();
}
return -1;
}
public long getMaxOffset() {
try {
this.readWriteLock.readLock().lock();
if (!this.mapedFiles.isEmpty()) {
int lastIndex = this.mapedFiles.size() - 1;
MapedFile mapedFile = this.mapedFiles.get(lastIndex);
return mapedFile.getFileFromOffset() + mapedFile.getWrotePostion();
}
}
catch (Exception e) {
log.error("getMinOffset has exception.", e);
}
finally {
this.readWriteLock.readLock().unlock();
}
return 0;
}
/**
* �ָ�ʱ����
*/
public void deleteLastMapedFile() {
if (!this.mapedFiles.isEmpty()) {
int lastIndex = this.mapedFiles.size() - 1;
MapedFile mapedFile = this.mapedFiles.get(lastIndex);
mapedFile.destroy(1000);
this.mapedFiles.remove(mapedFile);
log.info("on recover, destroy a logic maped file " + mapedFile.getFileName());
}
}
/**
* ɾ���ļ�ֻ�ܴ�ͷ��ʼɾ
*/
private void deleteExpiredFile(List<MapedFile> files) {
if (!files.isEmpty()) {
try {
this.readWriteLock.writeLock().lock();
for (MapedFile file : files) {
if (!this.mapedFiles.remove(file)) {
log.error("deleteExpiredFile remove failed.");
break;
}
}
}
catch (Exception e) {
log.error("deleteExpiredFile has exception.", e);
}
finally {
this.readWriteLock.writeLock().unlock();
}
}
}
/**
* �����ļ�����ʱ����ɾ����������ļ�
*/
public int deleteExpiredFileByTime(final long expiredTime, final int deleteFilesInterval,
final long intervalForcibly) {
Object[] mfs = this.copyMapedFiles(0);
if (null == mfs)
return 0;
// ���һ���ļ�����д״̬������ɾ��
int mfsLength = mfs.length - 1;
int deleteCount = 0;
List<MapedFile> files = new ArrayList<MapedFile>();
if (null != mfs) {
for (int i = 0; i < mfsLength; i++) {
MapedFile mapedFile = (MapedFile) mfs[i];
long liveMaxTimestamp = mapedFile.getLastModifiedTimestamp() + expiredTime;
if (System.currentTimeMillis() >= liveMaxTimestamp) {
if (mapedFile.destroy(intervalForcibly)) {
files.add(mapedFile);
deleteCount++;
if (files.size() >= DeleteFilesBatchMax) {
break;
}
if (deleteFilesInterval > 0 && (i + 1) < mfsLength) {
try {
Thread.sleep(deleteFilesInterval);
}
catch (InterruptedException e) {
e.printStackTrace();
}
}
}
else {
break;
}
}
}
}
deleteExpiredFile(files);
return deleteCount;
}
/**
* �������������СOffset��ɾ��������
*
* @param offset
* ���������Сoffset
*/
public int deleteExpiredFileByOffset(long offset, int unitSize) {
Object[] mfs = this.copyMapedFiles(0);
List<MapedFile> files = new ArrayList<MapedFile>();
int deleteCount = 0;
if (null != mfs) {
// ���һ���ļ�����д״̬������ɾ��
int mfsLength = mfs.length - 1;
// ���������Χ 0 ... last - 1
for (int i = 0; i < mfsLength; i++) {
boolean destroy = true;
MapedFile mapedFile = (MapedFile) mfs[i];
SelectMapedBufferResult result = mapedFile.selectMapedBuffer(this.mapedFileSize - unitSize);
if (result != null) {
long maxOffsetInLogicQueue = result.getByteBuffer().getLong();
result.release();
// ��ǰ�ļ��Ƿ����ɾ��
destroy = (maxOffsetInLogicQueue < offset);
if (destroy) {
log.info("physic min offset " + offset + ", logics in current mapedfile max offset "
+ maxOffsetInLogicQueue + ", delete it");
}
}
else {
log.warn("this being not excuted forever.");
break;
}
if (destroy && mapedFile.destroy(1000 * 60)) {
files.add(mapedFile);
deleteCount++;
}
else {
break;
}
}
}
deleteExpiredFile(files);
return deleteCount;
}
/**
* ����ֵ��ʾ�Ƿ�ȫ��ˢ�����
*
* @return
*/
public boolean commit(final int flushLeastPages) {
boolean result = true;
MapedFile mapedFile = this.findMapedFileByOffset(this.committedWhere, true);
if (mapedFile != null) {
long tmpTimeStamp = mapedFile.getStoreTimestamp();
int offset = mapedFile.commit(flushLeastPages);
long where = mapedFile.getFileFromOffset() + offset;
result = (where == this.committedWhere);
this.committedWhere = where;
if (0 == flushLeastPages) {
this.storeTimestamp = tmpTimeStamp;
}
}
return result;
}
public MapedFile getLastMapedFile2() {
if (this.mapedFiles.isEmpty()) {
return null;
}
return this.mapedFiles.get(this.mapedFiles.size() - 1);
}
public MapedFile getLastMapedFile() {
return this.getLastMapedFile(0);
}
/**
* ��ȡ���һ��MapedFile�������һ����û�У����´���һ����������һ��д���ˣ����´���һ��
*
* @param startOffset
* ��������µ��ļ�����ʼoffset
* @return
*/
public MapedFile getLastMapedFile(final long startOffset) {
long createOffset = -1;
MapedFile mapedFileLast = null;
{
this.readWriteLock.readLock().lock();
if (this.mapedFiles.isEmpty()) {
createOffset = startOffset - (startOffset % this.mapedFileSize);
}
else {
mapedFileLast = this.mapedFiles.get(this.mapedFiles.size() - 1);
}
this.readWriteLock.readLock().unlock();
}
if (mapedFileLast != null && mapedFileLast.isFull()) {
createOffset = mapedFileLast.getFileFromOffset() + this.mapedFileSize;
}
if (createOffset != -1) {
String nextFilePath = this.storePath + File.separator + UtilALl.offset2FileName(createOffset);
String nextNextFilePath =
this.storePath + File.separator + UtilALl.offset2FileName(createOffset + this.mapedFileSize);
MapedFile mapedFile =
this.allocateMapedFileService.putRequestAndReturnMapedFile(nextFilePath, nextNextFilePath,
this.mapedFileSize);
if (mapedFile != null) {
this.readWriteLock.writeLock().lock();
if (this.mapedFiles.isEmpty()) {
mapedFile.setFirstCreateInQueue(true);
}
this.mapedFiles.add(mapedFile);
this.readWriteLock.writeLock().unlock();
}
return mapedFile;
}
return mapedFileLast;
}
private MapedFile getFirstMapedFile() {
if (this.mapedFiles.isEmpty()) {
return null;
}
return this.mapedFiles.get(0);
}
public MapedFile getFirstMapedFileOnLock() {
try {
this.readWriteLock.readLock().lock();
return this.getFirstMapedFile();
}
finally {
this.readWriteLock.readLock().unlock();
}
}
public MapedFile findMapedFileByOffset(final long offset) {
return findMapedFileByOffset(offset, false);
}
public MapedFile findMapedFileByOffset(final long offset, final boolean returnFirstOnNotFound) {
try {
this.readWriteLock.readLock().lock();
MapedFile mapedFile = this.getFirstMapedFile();
if (mapedFile != null) {
int index =
(int) ((offset / this.mapedFileSize) - (mapedFile.getFileFromOffset() / this.mapedFileSize));
if (index < 0 || index >= this.mapedFiles.size()) {
log.warn("findMapedFileByOffset offset not matched, request Offset: " + offset + ", index: "
+ index + ", mapedFileSize: " + this.mapedFileSize + ", mapedFiles count: "
+ this.mapedFiles.size());
}
try {
return this.mapedFiles.get(index);
}
catch (Exception e) {
if (returnFirstOnNotFound) {
return mapedFile;
}
}
}
}
catch (Exception e) {
e.printStackTrace();
}
finally {
this.readWriteLock.readLock().unlock();
}
return null;
}
public long getMapedMemorySize() {
long size = 0;
Object[] mfs = this.copyMapedFiles(0);
if (mfs != null) {
for (Object mf : mfs) {
if (((ReferenceResource) mf).isAvailable()) {
size += this.mapedFileSize;
}
}
}
return size;
}
private Object[] copyMapedFiles(final int reservedMapedFiles) {
Object[] mfs = null;
try {
this.readWriteLock.readLock().lock();
if (this.mapedFiles.size() <= reservedMapedFiles) {
return null;
}
mfs = this.mapedFiles.toArray();
}
catch (Exception e) {
e.printStackTrace();
}
finally {
this.readWriteLock.readLock().unlock();
}
return mfs;
}
public boolean retryDeleteFirstFile(final long intervalForcibly) {
MapedFile mapedFile = this.getFirstMapedFileOnLock();
if (mapedFile != null) {
if (!mapedFile.isAvailable()) {
log.warn("the mapedfile was destroyed once, but still alive, " + mapedFile.getFileName());
boolean result = mapedFile.destroy(intervalForcibly);
if (result) {
log.warn("the mapedfile redelete OK, " + mapedFile.getFileName());
List<MapedFile> tmps = new ArrayList<MapedFile>();
tmps.add(mapedFile);
this.deleteExpiredFile(tmps);
}
else {
log.warn("the mapedfile redelete Failed, " + mapedFile.getFileName());
}
return result;
}
}
return false;
}
/**
* �رն��У��������ݻ��ڣ����Dz��ܷ���
*/
public void shutdown(final long intervalForcibly) {
this.readWriteLock.readLock().lock();
for (MapedFile mf : this.mapedFiles) {
mf.shutdown(intervalForcibly);
}
this.readWriteLock.readLock().unlock();
}
/**
* ���ٶ��У��������ݱ�ɾ�����˺����п��ܲ��ɹ�
*/
public void destroy() {
this.readWriteLock.writeLock().lock();
for (MapedFile mf : this.mapedFiles) {
mf.destroy(1000 * 3);
}
this.mapedFiles.clear();
this.committedWhere = 0;
this.readWriteLock.writeLock().unlock();
}
public long getCommittedWhere() {
return committedWhere;
}
public void setCommittedWhere(long committedWhere) {
this.committedWhere = committedWhere;
}
public long getStoreTimestamp() {
return storeTimestamp;
}
public List<MapedFile> getMapedFiles() {
return mapedFiles;
}
public int getMapedFileSize() {
return mapedFileSize;
}
}