package io.urmia.st;
/**
*
* Copyright 2014 by Amin Abbaspour
*
* This file is part of Urmia.io
*
* Urmia.io is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Urmia.io is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Urmia.io. If not, see <http://www.gnu.org/licenses/>.
*/
import com.google.common.base.Joiner;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.*;
import io.netty.handler.codec.http.*;
import io.netty.util.CharsetUtil;
import io.urmia.util.AccessLog;
import io.urmia.util.DigestUtils;
import io.urmia.util.FileTime;
import io.urmia.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.activation.MimetypesFileTypeMap;
import java.io.*;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static io.netty.handler.codec.http.HttpHeaders.Names.*;
import static io.netty.handler.codec.http.HttpHeaders.isKeepAlive;
import static io.netty.handler.codec.http.HttpHeaders.setContentLength;
import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
import static io.netty.handler.codec.http.HttpResponseStatus.OK;
import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
public class StorageServerHandler extends SimpleChannelInboundHandler<HttpObject> {
private static final Logger log = LoggerFactory.getLogger(StorageServerHandler.class);
private static final AccessLog access = new AccessLog();
private final String BASE;
private HttpRequest request;
boolean readingChunks = false;
FileChannel fileChannel = null;
private volatile long requestStartMS;
private volatile String uri;
public StorageServerHandler(String BASE) {
this.BASE = BASE;
}
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
super.channelActive(ctx);
ctx.read();
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
log.error("exceptionCaught", cause);
}
@Override
protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception {
if (msg instanceof HttpRequest) {
requestStartMS = System.currentTimeMillis();
request = (HttpRequest) msg;
log.info("received HttpRequest: {} {}", request.getMethod(), request.getUri());
// todo: what's this?
//if (is100ContinueExpected(request)) {
// send100Continue(ctx);
//}
if (HttpMethod.PUT.equals(request.getMethod())) {
// start doing the fs put. next msg is not a LastHttpContent
handlePUT(ctx, request);
//return;
}
ctx.read();
return;
}
if (msg instanceof HttpContent) {
// New chunk is received
HttpContent chunk = (HttpContent) msg;
//log.info("chunk {} of size: {}", chunk, chunk.content().readableBytes());
if (fileChannel != null) {
writeToFile(chunk.content());
}
// example of reading only if at the end
if (chunk instanceof LastHttpContent) {
log.trace("received LastHttpContent: {}", chunk);
if (HttpMethod.HEAD.equals(request.getMethod())) {
handleHEAD(ctx, request);
ctx.read();
return;
}
if (HttpMethod.GET.equals(request.getMethod())) {
handleGET(ctx, request);
ctx.read();
return;
}
if (HttpMethod.DELETE.equals(request.getMethod())) {
handleDELETE(ctx, request);
ctx.read();
return;
}
if (HttpMethod.PUT.equals(request.getMethod())) {
// TODO: reset() if exception catch or timeout (i.e. no LastHttpContent)
writeResponse(ctx, new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK), true); // close the connection after upload (mput) done
reset();
access.success(ctx, "PUT", uri, requestStartMS);
ctx.read();
return;
}
log.warn("unknown request: {}", request);
sendError(ctx, HttpResponseStatus.BAD_REQUEST);
}
ctx.read();
return;
}
log.warn("unknown msg type: {}", msg);
}
private void handleGET(ChannelHandlerContext ctx, HttpRequest request) throws IOException {
log.info("handleGET req: {}", request);
final String uri = request.getUri();
// todo: handle args: limit, object=true, directory=true, marker=xyz, sort_order=reverse, sort=mtime
final int queryPos = uri.lastIndexOf('?');
final String uriNoArgs = queryPos == -1 ? uri : uri.substring(0, queryPos);
log.info("GET uriNoArgs: {}", uriNoArgs);
final String path = BASE + uriNoArgs;
File file = new File(path);
if (file.exists()) {
if (file.isFile()) {
downloadFile(ctx, file);
return;
}
if (file.isDirectory()) {
listDirectory(ctx, file);
return;
}
}
sendError(ctx, NOT_FOUND);
}
private ByteBuf errorBody(String code, String message) {
Map<String, Object> m = new HashMap<String, Object>(2);
m.put("code", code);
m.put("message", message);
String json = StringUtils.mapToJson(m) + "\n";
return Unpooled.copiedBuffer(json, CharsetUtil.UTF_8);
}
private void handleDELETE(ChannelHandlerContext ctx, HttpRequest request) throws IOException {
log.info("handleDELETE req: {}", request);
final String uri = request.getUri();
final int queryPos = uri.lastIndexOf('?');
final String uriNoArgs = queryPos == -1 ? uri : uri.substring(0, queryPos);
log.info("DELETE uriNoArgs: {}", uriNoArgs);
final String path = BASE + uriNoArgs;
File file = new File(path);
if (!file.exists()) {
sendError(ctx, HttpResponseStatus.NOT_FOUND);
access.success(ctx, "DELETE", uri, requestStartMS);
return;
}
if (file.isDirectory()) {
File[] files = file.listFiles();
if (files != null && files.length != 0) {
writeResponse(ctx,
new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST,
errorBody("DirectoryNotEmptyError", uriNoArgs + " is not empty")), true);
//sendError(ctx, HttpResponseStatus.BAD_REQUEST); // dir not empty
return;
}
}
if (file.delete()) {
log.info("deleted: {}", path);
writeResponse(ctx, new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NO_CONTENT), true);
access.success(ctx, "DELETE", uri + " -> " + file.getAbsolutePath(), requestStartMS);
} else {
log.info("unable to delete: {}", path);
sendError(ctx, HttpResponseStatus.BAD_REQUEST);
access.fail(ctx, "DELETE", uri, requestStartMS);
}
}
private static final Joiner LINE_JOINER = Joiner.on('\n').skipNulls();
private void listDirectory(ChannelHandlerContext ctx, File file) throws IOException {
log.info("listDirectory: {}", file);
File[] files = file.listFiles();
if (files == null) {
ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
return;
}
List<String> jsons = new ArrayList<String>(files.length);
for (File f : files) {
log.debug("found: {}", f);
jsons.add(toJson(f));
}
String result = LINE_JOINER.join(jsons);
log.info("writing back: {}", result);
HttpResponseStatus status = HttpResponseStatus.OK;
FullHttpResponse response = new DefaultFullHttpResponse(
HTTP_1_1, status, Unpooled.copiedBuffer(result, CharsetUtil.UTF_8));
response.headers().set(CONTENT_TYPE, "application/x-json-stream; type=directory");
response.headers().set(CONTENT_LENGTH, result.length());
response.headers().set("result-set-size", files.length);
// Write the end marker
ChannelFuture lastContentFuture = ctx.writeAndFlush(response);
// Decide whether to close the connection or not.
if (!isKeepAlive(request)) {
// Close the connection when the whole content is written out.
lastContentFuture.addListener(ChannelFutureListener.CLOSE);
}
access.success(ctx, "LIST", file.getAbsolutePath(), requestStartMS);
}
private String toJson(File f) throws IOException {
Map<String, Object> map = new HashMap<String, Object>(6);
//Path path = f.toPath();
map.put("name", f.getName());
//TODO map.put("etag", calcETAG(f));
map.put("size", f.length());
map.put("type", f.isDirectory() ? "directory" : "object");
map.put("mtime", FileTime.fromMillis(f.lastModified()));
map.put("durability", 1);
return StringUtils.mapToJson(map);
}
private void downloadFile(ChannelHandlerContext ctx, File file) throws IOException {
final RandomAccessFile raf;
try {
raf = new RandomAccessFile(file, "r");
} catch (FileNotFoundException fnfe) {
log.warn("no file at: {}", file.getPath());
access.fail(ctx, "GET", file.getAbsolutePath(), requestStartMS);
sendError(ctx, NOT_FOUND);
return;
}
final long fileLength = raf.length();
log.info("downloading file: {} of len: {}", file, fileLength);
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
setContentLength(response, fileLength);
setContentTypeHeader(response, file);
//setDateAndCacheHeaders(response, file);
if (isKeepAlive(request)) {
response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
}
// Write the initial line and the header.
ctx.write(response);
final ChannelFuture sendFileFuture;
sendFileFuture = ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise());
sendFileFuture.addListener(new ChannelProgressiveFutureListener() {
@Override
public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) {
if (total < 0) { // total unknown
System.err.println("Transfer progress: " + progress);
} else {
System.err.println("Transfer progress: " + progress + " / " + total);
}
}
@Override
public void operationComplete(ChannelProgressiveFuture future) throws Exception {
System.err.println("Transfer complete.");
}
});
// Write the end marker
ChannelFuture lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
access.success(ctx, "GET", file.getAbsolutePath(), requestStartMS);
// Decide whether to close the connection or not.
if (!isKeepAlive(request)) {
// Close the connection when the whole content is written out.
lastContentFuture.addListener(ChannelFutureListener.CLOSE);
}
}
/**
* Sets the content type header for the HTTP Response
*
* @param response HTTP response
* @param file file to extract content type
*/
private static void setContentTypeHeader(HttpResponse response, File file) {
MimetypesFileTypeMap mimeTypesMap = new MimetypesFileTypeMap();
response.headers().set(CONTENT_TYPE, mimeTypesMap.getContentType(file.getPath()));
}
private static void sendError(ChannelHandlerContext ctx, HttpResponseStatus status) {
FullHttpResponse response = new DefaultFullHttpResponse(
HTTP_1_1, status, Unpooled.copiedBuffer("Failure: " + status.toString() + "\r\n", CharsetUtil.UTF_8));
response.headers().set(CONTENT_TYPE, "text/plain; charset=UTF-8");
// Close the connection as soon as the error message is sent.
ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
}
private void writeToFile(ByteBuf buf) throws IOException {
if (fileChannel == null)
return;
fileChannel.write(buf.nioBuffers());
}
private void reset() throws IOException {
request = null;
readingChunks = false;
if (fileChannel != null) {
fileChannel.close();
fileChannel = null;
}
}
private void writeResponse(ChannelHandlerContext ctx, FullHttpResponse response, boolean forceClose) {
// Decide whether to close the connection or not.
boolean close = forceClose
|| HttpHeaders.Values.CLOSE.equalsIgnoreCase(request.headers().get(HttpHeaders.Names.CONNECTION))
|| request.getProtocolVersion().equals(HttpVersion.HTTP_1_0)
&& !HttpHeaders.Values.KEEP_ALIVE.equalsIgnoreCase(request.headers().get(HttpHeaders.Names.CONNECTION));
log.debug("writeResponse close: {}, data: {}", close, response);
ChannelFuture future = ctx.writeAndFlush(response);
// Close the connection after the write operation is done if necessary.
if (close) {
future.addListener(ChannelFutureListener.CLOSE);
}
}
private void handleHEAD(ChannelHandlerContext ctx, HttpRequest request) throws IOException {
log.info("handleHEAD uri: {} ", request.getUri());
HttpResponseStatus status = HttpResponseStatus.NO_CONTENT;
FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, status);
String uri = request.getUri(); // HEAD /abbaspour/stor/20120624_002h.jpg HTTP/1.1
final String path = BASE + uri;
File f = new File(path);
if (f.exists()) {
if (f.isDirectory()) {
response.headers().set(CONTENT_TYPE, "application/x-json-stream; type=directory");
} else {
setContentTypeHeader(response, f);
response.headers().set(CONTENT_MD5, DigestUtils.md5sum(f));
}
}
access.success(ctx, "HEAD", request.getUri(), requestStartMS);
ctx.writeAndFlush(response); // VoidChannelPromise
}
private static boolean isDirectory(String contentType) {
return contentType != null && contentType.endsWith("; type=directory");
}
private void handlePUT(ChannelHandlerContext ctx, HttpRequest request) /*throws IOException*/ {
log.debug("handlePUT: {}", request);
final String location = getLocation(request);
String uri = request.getUri();
if (location != null) {
String existing = BASE + location;
String link = BASE + uri;
log.debug("ln {} -> {}", link, existing);
link(ctx, existing, link);
return;
}
boolean isDir = isDirectory(request.headers().get("content-type"));
// PUT
String p = BASE + uri;
log.info("creating {} at: {}", isDir ? "dir" : "file", p);
File file = new File(p);
if (isDir) {
log.info("mkdir at: {}", p);
final boolean done = file.mkdirs();
if (done) {
access.success(ctx, "MKDIR", uri, requestStartMS);
ctx.writeAndFlush(new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NO_CONTENT));
} else {
access.fail(ctx, "MKDIR", uri, requestStartMS);
sendError(ctx, HttpResponseStatus.BAD_REQUEST);
}
} else {
file.getParentFile().mkdirs();
this.uri = uri;
// todo: handle 'location' header for 'mln()'. e.i. no content
//Path path = file.toPath();
//fileChannel = FileChannel.open(path, CREATE, WRITE);
try {
fileChannel = new FileOutputStream(file).getChannel(); // todo: should close fis?
} catch (FileNotFoundException e) {
log.error("exception in opening file channel: {}, err: {}", file, e.getMessage());
writeResponse(ctx,
new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST,
errorBody("UploadError", "invalid path: " + uri)), true);
return;
}
readingChunks = HttpHeaders.isTransferEncodingChunked(request);
//log.info("is chunk: {}", readingChunks);
ctx.writeAndFlush(new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.CONTINUE)); // VoidChannelPromise
}
}
String getLocation(HttpRequest request) {
return request.headers().get("location");
}
void link(ChannelHandlerContext ctx, String existing, String link) {
File existingFile = new File(existing);
if (!existingFile.exists()) {
log.warn("does not exist: {}", existing);
sendError(ctx, HttpResponseStatus.NOT_FOUND);
return;
}
File destFile = new File(link);
if(destFile.exists()) {
log.warn("destination already exist: {}", link);
sendError(ctx, HttpResponseStatus.BAD_REQUEST);
return;
}
try {
copyFile(existingFile, destFile);
} catch (IOException e) {
log.warn("error on link {} -> {}. error: {}", link, existing, e.getMessage());
//sendError(ctx, HttpResponseStatus.BAD_REQUEST);
writeResponse(ctx,
new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST,
errorBody("LinkError", e.getClass().getSimpleName())), true);
}
writeResponse(ctx, new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NO_CONTENT), false);
}
private static void copyFile(File source, File dest) throws IOException {
FileChannel inputChannel = null;
FileChannel outputChannel = null;
try {
inputChannel = new FileInputStream(source).getChannel();
outputChannel = new FileOutputStream(dest).getChannel();
outputChannel.transferFrom(inputChannel, 0, inputChannel.size());
} finally {
if (inputChannel != null)
try {
inputChannel.close();
} catch (Exception ignored) {
}
if (outputChannel != null)
try {
outputChannel.close();
} catch (Exception ignored) {
}
}
}
}