/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.raid;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.raid.LogUtils.LOGRESULTS;
import org.apache.hadoop.raid.LogUtils.LOGTYPES;
import org.apache.hadoop.raid.RaidUtils.RaidInfo;
import org.apache.hadoop.raid.protocol.PolicyInfo;
/**
* Periodically delete orphaned parity files.
*/
public class PurgeMonitor implements Runnable {
public static final Log LOG = LogFactory.getLog(PurgeMonitor.class);
public static final long PURGE_MONITOR_SLEEP_TIME_DEFAULT = 10000L;
public static final String PURGE_MONITOR_SLEEP_TIME_KEY =
"hdfs.raid.purge.monitor.sleep";
volatile boolean running = true;
private Configuration conf;
private PlacementMonitor placementMonitor;
private int directoryTraversalThreads;
private boolean directoryTraversalShuffle;
private long purgeMonitorSleepTime = PURGE_MONITOR_SLEEP_TIME_DEFAULT;
AtomicLong entriesProcessed;
private final RaidNode raidNode;
private final static List<FileStatus> modifiedSource =
new ArrayList<FileStatus>();
public PurgeMonitor(Configuration conf,
PlacementMonitor placementMonitor,
final RaidNode raidNode) {
this.conf = conf;
this.placementMonitor = placementMonitor;
this.directoryTraversalShuffle =
conf.getBoolean(RaidNode.RAID_DIRECTORYTRAVERSAL_SHUFFLE, true);
this.directoryTraversalThreads =
conf.getInt(RaidNode.RAID_DIRECTORYTRAVERSAL_THREADS, 4);
this.purgeMonitorSleepTime =
conf.getLong(PURGE_MONITOR_SLEEP_TIME_KEY,
PURGE_MONITOR_SLEEP_TIME_DEFAULT);
this.entriesProcessed = new AtomicLong(0);
this.raidNode = raidNode;
}
/**
*/
public void run() {
while (running) {
try {
doPurge();
} catch (Exception e) {
LOG.error("PurgeMonitor error ", e);
} finally {
LOG.info("Purge parity files thread continuing to run...");
}
}
}
/**
* Traverse the parity destination directory, removing directories that
* no longer existing in the source.
* @throws IOException
*/
private void purgeDirectories(FileSystem fs, Path root) throws IOException {
DirectoryTraversal traversal =
DirectoryTraversal.directoryRetriever(Arrays.asList(root), fs,
directoryTraversalThreads, directoryTraversalShuffle);
String prefix = root.toUri().getPath();
FileStatus dir;
while ((dir = traversal.next()) != DirectoryTraversal.FINISH_TOKEN) {
Path dirPath = dir.getPath();
if (dirPath.toUri().getPath().endsWith(RaidNode.HAR_SUFFIX)) {
continue;
}
String dirStr = dirPath.toUri().getPath();
if (!dirStr.startsWith(prefix)) {
continue;
}
entriesProcessed.incrementAndGet();
String src = dirStr.replaceFirst(prefix, "");
if (src.length() == 0) continue;
Path srcPath = new Path(src);
if (!fs.exists(srcPath)) {
performDelete(fs, dirPath, true);
}
}
}
void doPurge() throws IOException, InterruptedException {
entriesProcessed.set(0);
while (running) {
Thread.sleep(purgeMonitorSleepTime);
placementMonitor.startCheckingFiles();
try {
// purge the directories
for (Codec c : Codec.getCodecs()) {
Path parityPath = new Path(c.parityDirectory);
FileSystem parityFs = parityPath.getFileSystem(conf);
// One pass to purge directories that dont exists in the src.
// This is cheaper than looking at parities.
LOG.info("Check directory for codec: " + c.id + ", directory: " + c.parityDirectory);
purgeDirectories(parityFs, parityPath);
}
// purge the parities.
for (Codec c : Codec.getCodecs()) {
modifiedSource.clear();
purgeCode(c);
try {
// re-generate the parity files for modified sources.
if (modifiedSource.size() > 0) {
LOG.info("re-generate parity files");
PolicyInfo info = raidNode.determinePolicy(c);
// check if we should raid the files/directories.
for (Iterator<FileStatus> it = modifiedSource.iterator();
it.hasNext();) {
FileStatus stat = it.next();
if (!RaidNode.shouldRaid(conf,
stat.getPath().getFileSystem(conf), stat, c)) {
it.remove();
}
}
raidNode.raidFiles(info, modifiedSource);
}
} catch (Exception ex) {
// ignore the error
LOG.warn(ex.getMessage(), ex);
}
}
} finally {
placementMonitor.clearAndReport();
}
}
}
void purgeCode(Codec codec) throws IOException {
Path parityPath = new Path(codec.parityDirectory);
FileSystem parityFs = parityPath.getFileSystem(conf);
PolicyInfo policy = raidNode == null ? null: raidNode.determinePolicy(codec);
FileSystem srcFs = parityFs; // Assume src == parity
FileStatus stat = null;
try {
stat = parityFs.getFileStatus(parityPath);
} catch (FileNotFoundException e) {}
if (stat == null) return;
LOG.info("Purging obsolete parity files for " + parityPath);
DirectoryTraversal obsoleteParityFileRetriever =
new DirectoryTraversal(
"Purge File ",
Collections.singletonList(parityPath),
parityFs,
new PurgeParityFileFilter(conf, codec, policy, srcFs, parityFs,
parityPath.toUri().getPath(), placementMonitor, entriesProcessed),
directoryTraversalThreads,
directoryTraversalShuffle);
FileStatus obsolete = null;
while ((obsolete = obsoleteParityFileRetriever.next()) !=
DirectoryTraversal.FINISH_TOKEN) {
performDelete(parityFs, obsolete.getPath(), false);
}
if (!codec.isDirRaid) {
DirectoryTraversal obsoleteParityHarRetriever =
new DirectoryTraversal(
"Purge HAR ",
Collections.singletonList(parityPath),
parityFs,
new PurgeHarFilter(conf, codec, policy, srcFs, parityFs,
parityPath.toUri().getPath(), placementMonitor, entriesProcessed),
directoryTraversalThreads,
directoryTraversalShuffle);
while ((obsolete = obsoleteParityHarRetriever.next()) !=
DirectoryTraversal.FINISH_TOKEN) {
performDelete(parityFs, obsolete.getPath(), true);
}
}
}
static void performDelete(FileSystem fs, Path p, boolean recursive)
throws IOException {
DistributedFileSystem dfs = DFSUtil.convertToDFS(fs);
boolean success = dfs.delete(p, recursive);
if (success) {
LOG.info("Purging " + p + ", recursive=" + recursive);
RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID).entriesPurged.inc();
} else {
LOG.error("Could not delete " + p);
}
}
static class PurgeHarFilter implements DirectoryTraversal.Filter {
Configuration conf;
Codec codec;
PolicyInfo policy;
FileSystem srcFs;
FileSystem parityFs;
String parityPrefix;
PlacementMonitor placementMonitor;
AtomicLong counter;
PurgeHarFilter(
Configuration conf,
Codec codec,
PolicyInfo policy,
FileSystem srcFs,
FileSystem parityFs,
String parityPrefix,
PlacementMonitor placementMonitor,
AtomicLong counter) {
this.conf = conf;
this.codec = codec;
this.policy = policy;
this.parityPrefix = parityPrefix;
this.srcFs = srcFs;
this.parityFs = parityFs;
this.placementMonitor = placementMonitor;
this.counter = counter;
}
@Override
public boolean check(FileStatus f) throws IOException {
if (f.isDir()) {
String pathStr = f.getPath().toUri().getPath();
if (pathStr.endsWith(RaidNode.HAR_SUFFIX)) {
counter.incrementAndGet();
try {
float harUsedPercent =
usefulHar(codec, policy, srcFs, parityFs, f.getPath(),
parityPrefix, conf, placementMonitor);
LOG.info("Useful percentage of " + pathStr + " " + harUsedPercent);
// Delete the har if its usefulness reaches a threshold.
if (harUsedPercent <= conf.getFloat("raid.har.usage.threshold", 0)) {
return true;
}
} catch (IOException e) {
LOG.warn("Error in har check ", e);
}
}
}
return false;
}
}
static class PurgeParityFileFilter implements DirectoryTraversal.Filter {
Configuration conf;
Codec codec;
PolicyInfo policy;
FileSystem srcFs;
FileSystem parityFs;
String parityPrefix;
PlacementMonitor placementMonitor;
AtomicLong counter;
final long minFileSize;
PurgeParityFileFilter(
Configuration conf,
Codec codec,
PolicyInfo policy,
FileSystem srcFs,
FileSystem parityFs,
String parityPrefix,
PlacementMonitor placementMonitor,
AtomicLong counter) {
this.conf = conf;
this.codec = codec;
this.policy = policy;
this.parityPrefix = parityPrefix;
this.srcFs = srcFs;
this.parityFs = parityFs;
this.placementMonitor = placementMonitor;
this.counter = counter;
this.minFileSize = conf.getLong(RaidNode.MINIMUM_RAIDABLE_FILESIZE_KEY,
RaidNode.MINIMUM_RAIDABLE_FILESIZE);
}
private void checkSrcDir(FileSystem srcFs, FileStatus dirStat) throws IOException {
if (!dirStat.isDir()) {
return;
}
if (placementMonitor == null) {
LOG.warn("PlacementMonitor is null, can not check the file.");
return;
}
FileStatus[] files = srcFs.listStatus(dirStat.getPath());
for (FileStatus stat : files) {
// only check small unraided files.
if (stat.getLen() >= this.minFileSize) {
continue;
}
placementMonitor.checkSrcFile(srcFs, stat);
}
}
@Override
public boolean check(FileStatus f) throws IOException {
if (f.isDir()) return false;
String pathStr = f.getPath().toUri().getPath();
// Verify the parityPrefix is a prefix of the parityPath
if (!pathStr.startsWith(parityPrefix)) return false;
// Do not deal with parity HARs here.
if (pathStr.indexOf(RaidNode.HAR_SUFFIX) != -1) return false;
counter.incrementAndGet();
String src = pathStr.replaceFirst(parityPrefix, "");
Path srcPath = new Path(src);
boolean shouldDelete = false;
FileStatus srcStat = null;
try {
srcStat = srcFs.getFileStatus(srcPath);
} catch (FileNotFoundException e) {
// No such src file, delete the parity file.
shouldDelete = true;
}
// check the src files of the directory-raid parity
if (srcStat != null && codec.isDirRaid) {
checkSrcDir(srcFs, srcStat);
}
if (!shouldDelete) {
try {
if (existsBetterParityFile(codec, srcStat, conf)) {
shouldDelete = true;
}
if (!shouldDelete) {
if (srcStat.getModificationTime() != f.getModificationTime()
&& codec.isDirRaid) {
modifiedSource.add(srcStat);
LogUtils.logEvent(srcFs, srcPath, LOGTYPES.MODIFICATION_TIME_CHANGE,
LOGRESULTS.NONE, codec, null, null, codec.id);
// delete the parity file after we finish the re-generation.
return false;
}
ParityFilePair ppair =
ParityFilePair.getParityFile(codec, srcStat, conf);
if ( ppair == null ||
!parityFs.equals(ppair.getFileSystem()) ||
!pathStr.equals(ppair.getPath().toUri().getPath())) {
shouldDelete = true;
} else {
// This parity file matches the source file.
if (placementMonitor != null) {
placementMonitor.checkFile(srcFs, srcStat,
ppair.getFileSystem(), ppair.getFileStatus(), codec, policy);
}
}
}
} catch (IOException e) {
LOG.warn("Error during purging " + src, e);
}
}
return shouldDelete;
}
}
/**
* Is there a parity file which has a codec with higher priority?
*/
private static boolean existsBetterParityFile(
Codec codec, FileStatus srcStat, Configuration conf) throws IOException {
for (Codec c : Codec.getCodecs()) {
if (c.priority > codec.priority) {
ParityFilePair ppair = ParityFilePair.getParityFile(
c, srcStat, conf);
if (ppair != null) {
return true;
}
}
}
return false;
}
//
// Returns the number of up-to-date files in the har as a percentage of the
// total number of files in the har.
//
protected static float usefulHar(
Codec codec,
PolicyInfo policy,
FileSystem srcFs, FileSystem parityFs,
Path harPath, String parityPrefix, Configuration conf,
PlacementMonitor placementMonitor) throws IOException {
HarIndex harIndex = HarIndex.getHarIndex(parityFs, harPath);
Iterator<HarIndex.IndexEntry> entryIt = harIndex.getEntries();
int numUseless = 0;
int filesInHar = 0;
while (entryIt.hasNext()) {
HarIndex.IndexEntry entry = entryIt.next();
filesInHar++;
if (!entry.fileName.startsWith(parityPrefix)) {
continue;
}
String src = entry.fileName.substring(parityPrefix.length());
Path srcPath = new Path(src);
FileStatus srcStatus = null;
try {
srcStatus = srcFs.getFileStatus(srcPath);
} catch (FileNotFoundException e) {
numUseless++;
continue;
}
if (existsBetterParityFile(codec, srcStatus, conf)) {
numUseless += 1;
continue;
}
try {
if (srcStatus == null) {
numUseless++;
} else if (entry.mtime != srcStatus.getModificationTime()) {
numUseless++;
} else {
// This parity file in this HAR is good.
if (placementMonitor != null) {
// Check placement.
placementMonitor.checkFile(
srcFs, srcStatus,
parityFs, harIndex.partFilePath(entry), entry, codec, policy);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Useful file " + entry.fileName);
}
}
} catch (FileNotFoundException e) {
numUseless++;
}
}
if (filesInHar == 0) { return 0; }
float uselessPercent = numUseless * 100.0f / filesInHar;
return 100 - uselessPercent;
}
public String htmlTable() {
return JspUtils.tableSimple(
JspUtils.tr(
JspUtils.td("Entries Processed") +
JspUtils.td(":") +
JspUtils.td(entriesProcessed.toString())));
}
}