/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.master.cleaner;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.backup.HFileArchiver;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.junit.Test;
import org.junit.experimental.categories.Category;
/**
* Test the HFileLink Cleaner.
* HFiles with links cannot be deleted until a link is present.
*/
@Category(SmallTests.class)
public class TestHFileLinkCleaner {
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@Test
public void testHFileLinkCleaning() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDataTestDir().toString());
conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
"org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner," +
"org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner");
Path rootDir = FSUtils.getRootDir(conf);
FileSystem fs = FileSystem.get(conf);
final String tableName = "test-table";
final String tableLinkName = "test-link";
final String hfileName = "1234567890";
final String familyName = "cf";
HRegionInfo hri = new HRegionInfo(Bytes.toBytes(tableName));
HRegionInfo hriLink = new HRegionInfo(Bytes.toBytes(tableLinkName));
Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
tableName, hri.getEncodedName(), familyName);
Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
tableLinkName, hriLink.getEncodedName(), familyName);
// Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf);
Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName);
fs.mkdirs(familyPath);
Path hfilePath = new Path(familyPath, hfileName);
fs.createNewFile(hfilePath);
// Create link to hfile
Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName,
hriLink.getEncodedName(), familyName);
fs.mkdirs(familyLinkPath);
HFileLink.create(conf, fs, familyLinkPath, hri, hfileName);
Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName);
assertTrue(fs.exists(linkBackRefDir));
FileStatus[] backRefs = fs.listStatus(linkBackRefDir);
assertEquals(1, backRefs.length);
Path linkBackRef = backRefs[0].getPath();
// Initialize cleaner
final long ttl = 1000;
conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
Server server = new DummyServer();
HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir);
// Link backref cannot be removed
Thread.sleep(ttl * 2);
cleaner.chore();
assertTrue(fs.exists(linkBackRef));
assertTrue(fs.exists(hfilePath));
// Link backref can be removed
fs.rename(new Path(rootDir, tableLinkName), new Path(archiveDir, tableLinkName));
Thread.sleep(ttl * 2);
cleaner.chore();
assertFalse("Link should be deleted", fs.exists(linkBackRef));
// HFile can be removed
Thread.sleep(ttl * 2);
cleaner.chore();
assertFalse("HFile should be deleted", fs.exists(hfilePath));
// Remove everything
for (int i = 0; i < 4; ++i) {
Thread.sleep(ttl * 2);
cleaner.chore();
}
assertFalse("HFile should be deleted", fs.exists(new Path(archiveDir, tableName)));
assertFalse("Link should be deleted", fs.exists(new Path(archiveDir, tableLinkName)));
cleaner.interrupt();
}
private static Path getFamilyDirPath (final Path rootDir, final String table,
final String region, final String family) {
return new Path(new Path(new Path(rootDir, table), region), family);
}
static class DummyServer implements Server {
@Override
public Configuration getConfiguration() {
return TEST_UTIL.getConfiguration();
}
@Override
public ZooKeeperWatcher getZooKeeper() {
try {
return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
@Override
public CatalogTracker getCatalogTracker() {
return null;
}
@Override
public ServerName getServerName() {
return new ServerName("regionserver,60020,000000");
}
@Override
public void abort(String why, Throwable e) {}
@Override
public boolean isAborted() {
return false;
}
@Override
public void stop(String why) {}
@Override
public boolean isStopped() {
return false;
}
}
}