/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs;
import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.TimeZone;
import javax.security.auth.login.LoginException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UnixUserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.StringUtils;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.XMLReaderFactory;
/** An implementation of a protocol for accessing filesystems over HTTP.
* The following implementation provides a limited, read-only interface
* to a filesystem over HTTP.
* @see org.apache.hadoop.hdfs.server.namenode.ListPathsServlet
* @see org.apache.hadoop.hdfs.server.namenode.FileDataServlet
*/
public class HftpFileSystem extends FileSystem {
static {
HttpURLConnection.setFollowRedirects(true);
}
protected InetSocketAddress nnAddr;
protected UserGroupInformation ugi;
/** The URI reference this FileSystem was initialized with **/
protected URI initializedWith;
public static final String HFTP_TIMEZONE = "UTC";
public static final String HFTP_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
public static final String CONTENT_LENGTH_FIELD = "Content-Length";
public static final String STRICT_CONTENT_LENGTH = "dfs.hftp.strictContentLength";
private boolean doStrictContentLengthCheck = false;
public static final SimpleDateFormat getDateFormat() {
final SimpleDateFormat df = new SimpleDateFormat(HFTP_DATE_FORMAT);
df.setTimeZone(TimeZone.getTimeZone(HFTP_TIMEZONE));
return df;
}
protected static final ThreadLocal<SimpleDateFormat> df =
new ThreadLocal<SimpleDateFormat>() {
protected SimpleDateFormat initialValue() {
return getDateFormat();
}
};
@Override
public void initialize(URI name, Configuration conf) throws IOException {
super.initialize(name, conf);
setConf(conf);
try {
this.ugi = UnixUserGroupInformation.login(conf, true);
} catch (LoginException le) {
throw new IOException(StringUtils.stringifyException(le));
}
initializedWith = name;
if (conf.getBoolean(FSConstants.CLIENT_CONFIGURATION_LOOKUP_DONE, false)) {
try {
initializedWith = new URI(conf.get(FileSystem.FS_DEFAULT_NAME_KEY));
} catch (URISyntaxException e) {
LOG.error(e);
}
}
nnAddr = NetUtils.createSocketAddr(name.toString());
doStrictContentLengthCheck = conf.getBoolean(STRICT_CONTENT_LENGTH, false);
}
@Override
public URI getUri() {
try {
return new URI("hftp", initializedWith.getAuthority(), null, null, null);
} catch (URISyntaxException e) {
return null;
}
}
/**
* Open an HTTP connection to the namenode to read file data and metadata.
* @param path The path component of the URL
* @param query The query component of the URL
*/
protected HttpURLConnection openConnection(String path, String query)
throws IOException {
try {
final URL url = new URI("http", null, nnAddr.getAddress().getHostAddress(),
nnAddr.getPort(), path, query, null).toURL();
if (LOG.isTraceEnabled()) {
LOG.trace("url=" + url);
}
return (HttpURLConnection)url.openConnection();
} catch (URISyntaxException e) {
throw (IOException)new IOException().initCause(e);
}
}
@Override
public FSDataInputStream open(Path f, int buffersize) throws IOException {
HftpInputStream hftpIn = new HftpInputStream(f);
return new FSDataInputStream(hftpIn) {
@Override
public boolean isUnderConstruction() {
return ((HftpInputStream)in).isUnderConstruction();
}
};
}
private class HftpInputStream extends FSInputStream {
private final Path file;
private InputStream in;
private long pos;
private boolean isUnderConstruction;
private long totalBytesRead = 0;
private long contentLength;
public HftpInputStream(Path file) throws IOException {
this.file = file;
openStream("/data" + file.toUri().getPath(), "ugi=" + ugi);
pos = 0;
}
public boolean isUnderConstruction() {
return isUnderConstruction;
}
@Override
public int read() throws IOException {
int ret = in.read();
if (ret >= 0) {
++pos;
totalBytesRead++;
} else if (totalBytesRead < contentLength) {
throw new EOFException(
String.format(
"premature end of stream: expected total %d bytes and got only %d",
pos, contentLength
));
}
return ret;
}
public int read(byte[] b, int off, int len) throws IOException {
int bytesRead = in.read(b, off, len);
if (bytesRead > 0) {
pos += bytesRead;
totalBytesRead += bytesRead;
} else if (bytesRead < 0) {
if (totalBytesRead < contentLength) {
throw new EOFException(
String.format(
"premature end of stream: expected total %d bytes and got only %d",
contentLength, pos
));
}
}
return bytesRead;
}
public void close() throws IOException {
in.close();
}
@Override
public void seek(long pos) throws IOException {
in.close();
openStream("/data" + file.toUri().getPath(),
"seek=" + pos + "&ugi=" + ugi);
this.pos = pos;
}
@Override
public long getPos() throws IOException {
return pos;
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
return false;
}
@Override
public int available() {
return (int)
Math.max(Math.min(contentLength - pos, Integer.MAX_VALUE), 0);
}
private void openStream(String path, String query) throws IOException {
HttpURLConnection connection = openConnection(path, query);
connection.setRequestMethod("GET");
connection.connect();
String contentLengthStr =
connection.getHeaderField(HftpFileSystem.CONTENT_LENGTH_FIELD);
totalBytesRead = 0;
if (contentLengthStr != null) {
contentLength = Long.valueOf(
contentLengthStr
);
} else {
if (HftpFileSystem.this.doStrictContentLengthCheck) {
throw new IOException("missing require Content-Length header");
} else {
contentLength = -1; //unknown
}
}
isUnderConstruction =
"true".equals(connection.getHeaderField("isUnderConstruction"));
in = connection.getInputStream();
}
}
/** Class to parse and store a listing reply from the server. */
class LsParser extends DefaultHandler {
ArrayList<FileStatus> fslist = new ArrayList<FileStatus>();
public void startElement(String ns, String localname, String qname,
Attributes attrs) throws SAXException {
if ("listing".equals(qname)) return;
if (!"file".equals(qname) && !"directory".equals(qname)) {
if (RemoteException.class.getSimpleName().equals(qname)) {
throw new SAXException(RemoteException.valueOf(attrs));
}
throw new SAXException("Unrecognized entry: " + qname);
}
long modif;
long atime = 0;
try {
final SimpleDateFormat ldf = df.get();
modif = ldf.parse(attrs.getValue("modified")).getTime();
String astr = attrs.getValue("accesstime");
if (astr != null) {
atime = ldf.parse(astr).getTime();
}
} catch (ParseException e) { throw new SAXException(e); }
FileStatus fs = "file".equals(qname)
? new FileStatus(
Long.valueOf(attrs.getValue("size")).longValue(), false,
Short.valueOf(attrs.getValue("replication")).shortValue(),
Long.valueOf(attrs.getValue("blocksize")).longValue(),
modif, atime, FsPermission.valueOf(attrs.getValue("permission")),
attrs.getValue("owner"), attrs.getValue("group"),
new Path(getUri().toString(), attrs.getValue("path"))
.makeQualified(HftpFileSystem.this))
: new FileStatus(0L, true, 0, 0L,
modif, atime, FsPermission.valueOf(attrs.getValue("permission")),
attrs.getValue("owner"), attrs.getValue("group"),
new Path(getUri().toString(), attrs.getValue("path"))
.makeQualified(HftpFileSystem.this));
fslist.add(fs);
}
private void fetchList(String path, boolean recur) throws IOException {
try {
XMLReader xr = XMLReaderFactory.createXMLReader();
xr.setContentHandler(this);
HttpURLConnection connection = openConnection("/listPaths" + path,
"ugi=" + ugi + (recur? "&recursive=yes" : ""));
connection.setRequestMethod("GET");
connection.connect();
InputStream resp = connection.getInputStream();
xr.parse(new InputSource(resp));
} catch(SAXException e) {
final Exception embedded = e.getException();
if (embedded != null && embedded instanceof IOException) {
throw (IOException)embedded;
}
throw new IOException("invalid xml directory content", e);
}
}
public FileStatus getFileStatus(Path f) throws IOException {
fetchList(f.toUri().getPath(), false);
if (fslist.size() == 0) {
throw new FileNotFoundException("File does not exist: " + f);
}
return fslist.get(0);
}
public FileStatus[] listStatus(Path f, boolean recur) throws IOException {
fetchList(f.toUri().getPath(), recur);
if (fslist.size() > 0 && (fslist.size() != 1 || fslist.get(0).isDir())) {
fslist.remove(0);
}
return fslist.toArray(new FileStatus[0]);
}
public FileStatus[] listStatus(Path f) throws IOException {
return listStatus(f, false);
}
}
@Override
public FileStatus[] listStatus(Path f) throws IOException {
LsParser lsparser = new LsParser();
return lsparser.listStatus(f);
}
@Override
public FileStatus getFileStatus(Path f) throws IOException {
LsParser lsparser = new LsParser();
return lsparser.getFileStatus(f);
}
private class ChecksumParser extends DefaultHandler {
private FileChecksum filechecksum;
/** {@inheritDoc} */
public void startElement(String ns, String localname, String qname,
Attributes attrs) throws SAXException {
if (!MD5MD5CRC32FileChecksum.class.getName().equals(qname)) {
if (RemoteException.class.getSimpleName().equals(qname)) {
throw new SAXException(RemoteException.valueOf(attrs));
}
throw new SAXException("Unrecognized entry: " + qname);
}
filechecksum = MD5MD5CRC32FileChecksum.valueOf(attrs);
}
private FileChecksum getFileChecksum(String f) throws IOException {
final HttpURLConnection connection = openConnection(
"/fileChecksum" + f, "ugi=" + ugi);
try {
final XMLReader xr = XMLReaderFactory.createXMLReader();
xr.setContentHandler(this);
connection.setRequestMethod("GET");
connection.connect();
xr.parse(new InputSource(connection.getInputStream()));
} catch(SAXException e) {
final Exception embedded = e.getException();
if (embedded != null && embedded instanceof IOException) {
throw (IOException)embedded;
}
throw new IOException("invalid xml directory content", e);
} finally {
connection.disconnect();
}
return filechecksum;
}
}
/** {@inheritDoc} */
public FileChecksum getFileChecksum(Path f) throws IOException {
final String s = makeQualified(f).toUri().getPath();
return new ChecksumParser().getFileChecksum(s);
}
@Override
public Path getWorkingDirectory() {
return new Path("/").makeQualified(this);
}
@Override
public void setWorkingDirectory(Path f) { }
/** This optional operation is not yet supported. */
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
throw new IOException("Not supported");
}
@Override
public FSDataOutputStream create(Path f, FsPermission permission,
boolean overwrite, int bufferSize,
short replication, long blockSize,
Progressable progress) throws IOException {
throw new IOException("Not supported");
}
@Override
public boolean rename(Path src, Path dst) throws IOException {
throw new IOException("Not supported");
}
@Override
/*
* @deprecated Use delete(path, boolean)
*/
@Deprecated
public boolean delete(Path f) throws IOException {
throw new IOException("Not supported");
}
@Override
public boolean delete(Path f, boolean recursive) throws IOException {
throw new IOException("Not supported");
}
@Override
public boolean mkdirs(Path f, FsPermission permission) throws IOException {
throw new IOException("Not supported");
}
}