/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfsproxy;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.util.Random;
import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.log4j.Level;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.ProxyUsers;
/**
* A JUnit test for HdfsProxy
*/
public class TestHdfsProxy extends TestCase {
static final Log LOG = LogFactory.getLog(TestHdfsProxy.class);
static final URI LOCAL_FS = URI.create("file:///");
private static final int NFILES = 10;
private static String TEST_ROOT_DIR = new Path(System.getProperty(
"test.build.data", "/tmp")).toString().replace(' ', '+');
/**
* class MyFile contains enough information to recreate the contents of a
* single file.
*/
private static class MyFile {
private static Random gen = new Random();
private static final int MAX_LEVELS = 3;
private static final int MAX_SIZE = 8 * 1024;
private static String[] dirNames = { "zero", "one", "two", "three", "four",
"five", "six", "seven", "eight", "nine" };
private final String name;
private int size = 0;
private long seed = 0L;
MyFile() {
this(gen.nextInt(MAX_LEVELS));
}
MyFile(int nLevels) {
String xname = "";
if (nLevels != 0) {
int[] levels = new int[nLevels];
for (int idx = 0; idx < nLevels; idx++) {
levels[idx] = gen.nextInt(10);
}
StringBuffer sb = new StringBuffer();
for (int idx = 0; idx < nLevels; idx++) {
sb.append(dirNames[levels[idx]]);
sb.append("/");
}
xname = sb.toString();
}
long fidx = gen.nextLong() & Long.MAX_VALUE;
name = xname + Long.toString(fidx);
reset();
}
void reset() {
final int oldsize = size;
do {
size = gen.nextInt(MAX_SIZE);
} while (oldsize == size);
final long oldseed = seed;
do {
seed = gen.nextLong() & Long.MAX_VALUE;
} while (oldseed == seed);
}
String getName() {
return name;
}
int getSize() {
return size;
}
long getSeed() {
return seed;
}
}
private static MyFile[] createFiles(URI fsname, String topdir)
throws IOException {
return createFiles(FileSystem.get(fsname, new Configuration()), topdir);
}
/**
* create NFILES with random names and directory hierarchies with random (but
* reproducible) data in them.
*/
private static MyFile[] createFiles(FileSystem fs, String topdir)
throws IOException {
Path root = new Path(topdir);
MyFile[] files = new MyFile[NFILES];
for (int i = 0; i < NFILES; i++) {
files[i] = createFile(root, fs);
}
return files;
}
private static MyFile createFile(Path root, FileSystem fs, int levels)
throws IOException {
MyFile f = levels < 0 ? new MyFile() : new MyFile(levels);
Path p = new Path(root, f.getName());
FSDataOutputStream out = fs.create(p);
byte[] toWrite = new byte[f.getSize()];
new Random(f.getSeed()).nextBytes(toWrite);
out.write(toWrite);
out.close();
FileSystem.LOG.info("created: " + p + ", size=" + f.getSize());
return f;
}
private static MyFile createFile(Path root, FileSystem fs) throws IOException {
return createFile(root, fs, -1);
}
private static boolean checkFiles(FileSystem fs, String topdir, MyFile[] files)
throws IOException {
return checkFiles(fs, topdir, files, false);
}
private static boolean checkFiles(FileSystem fs, String topdir,
MyFile[] files, boolean existingOnly) throws IOException {
Path root = new Path(topdir);
for (int idx = 0; idx < files.length; idx++) {
Path fPath = new Path(root, files[idx].getName());
try {
fs.getFileStatus(fPath);
FSDataInputStream in = fs.open(fPath);
byte[] toRead = new byte[files[idx].getSize()];
byte[] toCompare = new byte[files[idx].getSize()];
Random rb = new Random(files[idx].getSeed());
rb.nextBytes(toCompare);
assertEquals("Cannnot read file.", toRead.length, in.read(toRead));
in.close();
for (int i = 0; i < toRead.length; i++) {
if (toRead[i] != toCompare[i]) {
return false;
}
}
toRead = null;
toCompare = null;
} catch (FileNotFoundException fnfe) {
if (!existingOnly) {
throw fnfe;
}
}
}
return true;
}
/** delete directory and everything underneath it. */
private static void deldir(FileSystem fs, String topdir) throws IOException {
fs.delete(new Path(topdir), true);
}
/** verify hdfsproxy implements the hftp interface */
public void testHdfsProxyInterface() throws Exception {
// Test currently fails in Jenkins with
// "org.apache.hadoop.ipc.RemoteException: hudson is not allowed to impersonate hudson"
// TODO: Fix and restore test after 0.20-security-204 release.
//
// doTestHdfsProxyInterface();
}
/** verify hdfsproxy implements the hftp interface */
private void doTestHdfsProxyInterface() throws Exception {
MiniDFSCluster cluster = null;
HdfsProxy proxy = null;
try {
final UserGroupInformation CLIENT_UGI = UserGroupInformation.getCurrentUser();
final String testUser = CLIENT_UGI.getShortUserName();
final String testGroup = CLIENT_UGI.getGroupNames()[0];
final Configuration dfsConf = new Configuration();
dfsConf.set("hadoop.proxyuser." + testUser + ".groups", testGroup);
dfsConf.set("hadoop.proxyuser." + testGroup + ".hosts",
"127.0.0.1,localhost");
dfsConf.set("hadoop.proxyuser." + testUser + ".hosts",
"127.0.0.1,localhost");
dfsConf.set("hadoop.security.authentication", "simple");
//make sure server will look at the right config
ProxyUsers.refreshSuperUserGroupsConfiguration(dfsConf);
cluster = new MiniDFSCluster(dfsConf, 2, true, null);
cluster.waitActive();
final FileSystem localfs = FileSystem.get(LOCAL_FS, dfsConf);
final FileSystem hdfs = cluster.getFileSystem();
final Configuration proxyConf = new Configuration(false);
proxyConf.set("hdfsproxy.dfs.namenode.address", hdfs.getUri().getHost() +
":" + hdfs.getUri().getPort());
proxyConf.set("hdfsproxy.https.address", "localhost:0");
final String namenode = hdfs.getUri().toString();
if (namenode.startsWith("hdfs://")) {
MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR + "/srcdat");
hdfs.copyFromLocalFile
(new Path("file:///" + TEST_ROOT_DIR + "/srcdat"),
new Path(namenode + "/destdat" ));
assertTrue("Source and destination directories do not match.",
checkFiles(hdfs, "/destdat", files));
proxyConf.set("proxy.http.test.listener.addr", "localhost:0");
proxy = new HdfsProxy(proxyConf);
proxy.start();
InetSocketAddress proxyAddr = NetUtils.createSocketAddr("localhost:0");
final String realProxyAddr = proxyAddr.getHostName() + ":"
+ proxy.getPort();
final Path proxyUrl = new Path("hftp://" + realProxyAddr);
final FileSystem hftp = proxyUrl.getFileSystem(dfsConf);
FileUtil.copy(hftp, new Path(proxyUrl, "/destdat"),
hdfs, new Path(namenode + "/copied1"),
false, true, proxyConf);
assertTrue("Source and copied directories do not match.", checkFiles(
hdfs, "/copied1", files));
FileUtil.copy(hftp, new Path(proxyUrl, "/destdat"),
localfs, new Path(TEST_ROOT_DIR + "/copied2"),
false, true, proxyConf);
assertTrue("Source and copied directories do not match.", checkFiles(
localfs, TEST_ROOT_DIR + "/copied2", files));
deldir(hdfs, "/destdat");
deldir(hdfs, "/logs");
deldir(hdfs, "/copied1");
deldir(localfs, TEST_ROOT_DIR + "/srcdat");
deldir(localfs, TEST_ROOT_DIR + "/copied2");
}
if (cluster != null) {
cluster.shutdown();
}
if (proxy != null) {
proxy.stop();
}
} catch (Exception t) {
LOG.fatal("caught exception in test", t);
if (cluster != null) {
cluster.shutdown();
}
if (proxy != null) {
proxy.stop();
}
throw t;
}
}
}