/*******************************************************************************
* Copyright (c) 2008 Cambridge Semantics Incorporated.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
*
* Cambridge Semantics Incorporated - Initial Implementation
*******************************************************************************/
package org.openanzo.binarystore.server;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.Dictionary;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileUploadException;
import org.apache.commons.fileupload.ProgressListener;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.lang.StringUtils;
import org.eclipse.jetty.http.HttpContent;
import org.eclipse.jetty.http.HttpHeaders;
import org.json.JSONException;
import org.json.JSONWriter;
import org.openanzo.binarystore.BinaryStoreDictionary;
import org.openanzo.client.AnzoClient;
import org.openanzo.client.ClientGraph;
import org.openanzo.client.IStatementChannel;
import org.openanzo.client.pool.AnzoClientPool;
import org.openanzo.client.pool.RestrictedAnzoClient;
import org.openanzo.exceptions.AnzoException;
import org.openanzo.exceptions.ExceptionConstants;
import org.openanzo.exceptions.LogUtils;
import org.openanzo.exceptions.Messages;
import org.openanzo.ontologies.openanzo.NamedGraph;
import org.openanzo.rdf.Constants;
import org.openanzo.rdf.IAnzoGraph;
import org.openanzo.rdf.INamedGraph;
import org.openanzo.rdf.IStatementListener;
import org.openanzo.rdf.Literal;
import org.openanzo.rdf.RDFFormat;
import org.openanzo.rdf.Resource;
import org.openanzo.rdf.Statement;
import org.openanzo.rdf.URI;
import org.openanzo.rdf.Constants.GRAPHS;
import org.openanzo.rdf.utils.ReadWriteUtils;
import org.openanzo.rdf.utils.SerializationUtils;
import org.openanzo.rdf.utils.StatementUtils;
import org.openanzo.rdf.vocabulary.DC;
import org.openanzo.rdf.vocabulary.RDF;
import org.openanzo.services.BinaryStoreConstants;
import org.openanzo.services.ITracker;
import org.openanzo.servlet.ResourceServerServlet;
import org.openanzo.servlet.ServletDictionary;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
/**
* Binary Store Servlet
*
* @author Simon Martin ( <a href="mailto:simon@cambridgesemantics.com">simon@cambridgesemantics.com </a>)
*
*/
public class BinaryStoreServlet extends ResourceServerServlet implements BinaryStoreConstants {
private static final long serialVersionUID = 1L;
private static final Logger log = LoggerFactory.getLogger(BinaryStoreServlet.class);
private String serverNodeRootPath = null;
private String serverRootPath = null;
private String servletPath = null;
private String serverNode = null;
private static Integer directoryIncrementer = 0;
private static Boolean directoryIncrementerSet = false;
private static Calendar currentCal = null;
private DiskFileItemFactory factory = null;
private static final int MAX_MEMORY_SIZE = 1048576;
private long progressUpdateFrequency = 0;
private LockFileUpdater lockFileUpdater = null;
private String nodelockid = null;
private File varDirFile = null;
private String docRoot = null;
private AnzoClientPool clientPool = null;
private RestrictedAnzoClient anzoClient = null;
BinaryStoreServlet(AnzoClientPool clientPool) {
this.clientPool = clientPool;
}
@Override
public String getServletInfo() {
return "Binary Store Servlet";
}
void initialize(Dictionary<?, ?> configProperties) throws AnzoException {
this.serverNode = BinaryStoreDictionary.getServerNode(configProperties);
this.serverRootPath = BinaryStoreDictionary.getFileSystemRoot(configProperties);
this.docRoot = ServletDictionary.getDocRoot(configProperties);
if (docRoot.startsWith("./")) {
String root = System.getProperty("ANZO_HOME");
docRoot = root + docRoot.substring((root.endsWith("/") ? 2 : 1));
}
// this.loginPageURL=URIUtil.addPaths(context, this.loginPageURL);
// this.errorPageURL=URIUtil.addPaths(context, this.errorPageURL);
this.progressUpdateFrequency = BinaryStoreDictionary.getProgressUpdateFrequency(configProperties, Long.valueOf(0));
File f = new File(serverRootPath);
File serverNodeFile = new File(f, serverNode);
if (!serverNodeFile.isDirectory()) {
serverNodeFile.mkdirs();
}
serverNodeRootPath = serverNodeFile.getAbsolutePath();
varDirFile = new File(f, BINARYSTORE_VAR_DIRECTORY);
File varNodeDirFile = new File(varDirFile, serverNode);
//remove any old uuid for this process.
deleteDirectory(varNodeDirFile);
varNodeDirFile.mkdirs();
String lockid = UUID.randomUUID().toString();
nodelockid = LOCKFILE_PREFIX + serverNode + LOCKFILE_DELIMETER + lockid;
//create update thread to update process id in node var directory.
try {
lockFileUpdater = new LockFileUpdater(new File(varNodeDirFile, lockid).getAbsolutePath());
} catch (IOException e) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR, e);
}
lockFileUpdater.start();
try {
serverNodeRootPath = serverNodeFile.getCanonicalPath();
} catch (IOException e) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR, e);
}
currentCal = Calendar.getInstance();
factory = new DiskFileItemFactory();
factory.setSizeThreshold(MAX_MEMORY_SIZE);
setDirectoryIncrementer();
Thread initThread = new Thread() {
@Override
public void run() {
try {
anzoClient = clientPool.getAnzoClient(true, "BinaryStoreServlet");
anzoClient.getRealtimeUpdates().addTracker(null, RDF.TYPE, BINARYSTORE_ITEM_URI, null, new IStatementListener<ITracker>() {
public void statementsRemoved(ITracker source, Statement... stmts) {
for (Statement s : stmts) {
deleteFile(s.getNamedGraphUri(), -1);
}
}
public void statementsAdded(ITracker source, Statement... stmts) {
}
});
} catch (AnzoException ae) {
log.error(LogUtils.SERVER_INTERNAL_MARKER, Messages.formatString(ExceptionConstants.COMBUS.JMS_REGISTER_SELECTOR_ERROR, " for Binary Store Servlet"), ae);
}
}
};
initThread.start();
}
void stop(boolean bundleStopping) throws AnzoException {
lockFileUpdater.shutdown();
if (bundleStopping) {
try {
anzoClient.getRealtimeUpdates().removeTracker(null, RDF.TYPE, BINARYSTORE_ITEM_URI, null);
} catch (AnzoException ae) {
if (!bundleStopping) {
log.error(LogUtils.SERVER_INTERNAL_MARKER, Messages.formatString(ExceptionConstants.COMBUS.JMS_UNREGISTER_SELECTOR_ERROR, " for Binary Store Servlet"), ae);
} else {
log.debug(LogUtils.SERVER_INTERNAL_MARKER, Messages.formatString(ExceptionConstants.COMBUS.JMS_UNREGISTER_SELECTOR_ERROR, " for Binary Store Servlet"), ae);
}
}
}
clientPool.returnAnzoClient(anzoClient, bundleStopping);
}
void reset() throws AnzoException {
synchronized (directoryIncrementer) {
File dir = new File(serverNodeRootPath);
deleteDirectory(dir);
dir.mkdirs();
directoryIncrementer = 0;
}
}
private void setDirectoryIncrementer() {
synchronized (directoryIncrementer) {
if (!directoryIncrementerSet) {
Calendar cal = Calendar.getInstance();
int year = cal.get(Calendar.YEAR);
int month = (cal.get(Calendar.MONTH) + 1);
int day = cal.get(Calendar.DAY_OF_MONTH);
String fn = serverNodeRootPath + File.separator;
fn += year + File.separator;
fn += month + File.separator;
fn += day + File.separator;
File dir = new File(fn);
int inc = -1;
if (dir.isDirectory()) {
File[] files = dir.listFiles();
for (File f : files) {
try {
int i = Integer.parseInt(f.getName());
if (i > inc)
inc = i;
} catch (NumberFormatException e) {
}
}
}
synchronized (directoryIncrementerSet) {
if (inc != -1)
directoryIncrementer = inc;
directoryIncrementerSet = true;
}
}
}
}
@Override
protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
try {
if (req.getUserPrincipal() == null) {
resp.setHeader(AUTH_HEADER, String.valueOf(HttpServletResponse.SC_UNAUTHORIZED));
resp.sendError(HttpServletResponse.SC_UNAUTHORIZED);
return;
}
if (servletPath == null)
servletPath = req.getScheme() + "://" + req.getHeader("host") + req.getContextPath() + req.getServletPath();
String pathInfo = req.getPathInfo();
if (pathInfo.length() > 0 && pathInfo.startsWith("/"))
pathInfo = pathInfo.substring(1);
if (pathInfo.equals(NOOP)) {
// Handle the NO-OP operation which is used to check proper authentication by clients before uploading a big file.
// The 100-Continue HTTP dance unfortunately doesn't work too well due to poor support by clients and servers. So
// this NOOP operation is an alternative.
sendNOOPResponse(req, resp);
return;
}
//a pool of anzoClients
RestrictedAnzoClient ac = null;
String user = null;
try {
try {
ac = clientPool.getAnzoClient(true, "BinaryStoreOperation");
user = req.getUserPrincipal().getName();
RestrictedAnzoClient rac = ac;
rac.setServiceUser(user);
String runAsUser = req.getHeader(AUTHRUNAS_HEADER);
if (runAsUser != null && runAsUser.length() > 0) {
if (ac.getServicePrincipal().isSysadmin()) {
rac.setServiceUser(runAsUser);
}
}
} catch (AnzoException ae) {
MDC.put(LogUtils.REMOTE_ADDRESS, req.getRemoteAddr());
log.error(LogUtils.BINARY_MARKER, Messages.formatString(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR_PROCESSING_REQUEST), ae);
MDC.clear();
resp.setContentType(RDFFormat.JSON.getDefaultMIMEType());
resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
try {
SerializationUtils.writeExceptionJSON(ae, resp.getWriter());
} catch (JSONException jsone) {
log.debug(LogUtils.BINARY_MARKER, Messages.formatString(ExceptionConstants.IO.ERROR_SERIALIZING_JSON), jsone);
}
return;
}
if (pathInfo.equals(CREATE) || pathInfo.equals(UPDATE)) {
try {
createUpdate(ac, req, resp, pathInfo.equals(UPDATE));
} catch (AnzoException e) {
MDC.put(LogUtils.REMOTE_ADDRESS, req.getRemoteAddr());
MDC.put(LogUtils.USER, user);
log.info(LogUtils.BINARY_MARKER, Messages.formatString(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR_PROCESSING_REQUEST), e);
sendJSONError(req, resp, e);
MDC.clear();
return;
}
} else if (pathInfo.equals(READ)) {
String uri = req.getParameter(GRAPH);
if (uri == null) {
resp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
RequestDispatcher dispatcher = req.getRequestDispatcher(uri);
if (dispatcher != null) {
dispatcher.forward(req, resp);
return;
}
} else if (pathInfo.equals(DELETE)) {
try {
delete(ac, req, resp);
} catch (AnzoException e) {
MDC.put(LogUtils.REMOTE_ADDRESS, req.getRemoteAddr());
MDC.put(LogUtils.USER, user);
log.info(LogUtils.BINARY_MARKER, Messages.formatString(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR_PROCESSING_REQUEST), e);
sendJSONError(req, resp, e);
MDC.clear();
return;
}
} else {
String uri = req.getRequestURL().toString();
int rc = HttpServletResponse.SC_NOT_FOUND;
if (uri != null) {
try {
rc = read(ac, uri, req, resp);
} catch (AnzoException e) {
MDC.put(LogUtils.REMOTE_ADDRESS, req.getRemoteAddr());
MDC.put(LogUtils.USER, user);
log.info(LogUtils.BINARY_MARKER, Messages.formatString(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR_PROCESSING_REQUEST), e);
sendJSONError(req, resp, e);
MDC.clear();
return;
}
}
if (rc != HttpServletResponse.SC_OK)
resp.sendError(rc);
}
} finally {
if (ac != null) {
clientPool.returnAnzoClient(ac, true);
}
}
} catch (JSONException ae) {
MDC.put(LogUtils.REMOTE_ADDRESS, req.getRemoteAddr());
log.error(LogUtils.BINARY_MARKER, Messages.formatString(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR_PROCESSING_REQUEST), ae);
MDC.clear();
resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
resp.getWriter().write(ae.getMessage());
}
}
@SuppressWarnings( { "unchecked", "null" })
private void createUpdate(AnzoClient ac, HttpServletRequest req, HttpServletResponse resp, boolean update) throws ServletException, AnzoException, IOException, JSONException {
//check permissions before uploading the file.
URI updateURI = null;
if (update) {
String graphURI = req.getParameter(GRAPH);
if (graphURI == null) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_NO_GRAPHURI_SPECIFIED);
}
updateURI = Constants.valueFactory.createURI(graphURI);
if (!ac.namedGraphExists(updateURI)) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_UPDATE_FILEDOESNOTEXIST, updateURI.toString());
} else if (!ac.canAddToNamedGraph(updateURI) || (!ac.canRemoveFromNamedGraph(updateURI))) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_UPDATE_PERMISSION_DENIED, updateURI.toString());
}
} else {
if (!ac.canAddToNamedGraph(GRAPHS.GRAPHS_DATASET))
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_UPDATE_PERMISSION_DENIED, GRAPHS.GRAPHS_DATASET.toString());
}
String feedbackId = req.getParameter(FEEDBACK_ID);
if (!ServletFileUpload.isMultipartContent(req))
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_MULTIPART_FORM_REQUIRED);
IStatementChannel scg = null;
URI feedbackidURIg = null;
ServletFileUpload upload = new ServletFileUpload(factory);
if (feedbackId != null) {
//Create a progress listener
final URI feedbackURI = Constants.valueFactory.createURI(createFeedbackURI(ac));
final URI feedbackidURI = Constants.valueFactory.createURI(feedbackId);
try {
final IStatementChannel sc = ac.getStatementChannel(feedbackURI, AnzoClient.NON_REVISIONED_NAMED_GRAPH);
scg = sc;
feedbackidURIg = feedbackidURI;
ac.updateRepository();
ProgressListener progressListener = new ProgressListener() {
Date prevTime = null;
public void update(long pBytesRead, long pContentLength, int pItems) {
Date nowTime = new Date();
if ((pContentLength != pBytesRead) && (prevTime != null && (nowTime.getTime() < (prevTime.getTime() + progressUpdateFrequency)))) {
return;
}
prevTime = nowTime;
Map<String, Object> messageProperties = new HashMap<String, Object>();
Collection<Statement> statements = new HashSet<Statement>();
statements.add(Constants.valueFactory.createStatement(feedbackidURI, BINARYSTORE_ITEM_PROGRESS_JOB_URI, BINARYSTORE_ITEM_UPLOAD_JOB_URI, feedbackidURI));
statements.add(Constants.valueFactory.createStatement(feedbackidURI, BINARYSTORE_ITEM_PROGRESS_JOB_COMPLETE_URI, Constants.valueFactory.createLiteral(pContentLength), feedbackidURI));
statements.add(Constants.valueFactory.createStatement(feedbackidURI, BINARYSTORE_ITEM_PROGRESS_JOB_COMPLETED_URI, Constants.valueFactory.createLiteral(pBytesRead), feedbackidURI));
try {
sc.sendMessage(messageProperties, statements);
} catch (AnzoException e) {
log.error(LogUtils.BINARY_MARKER, Messages.formatString(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR_SENDING_PROGRESS), e);
}
}
};
upload.setProgressListener(progressListener);
} catch (AnzoException e) {
log.error(LogUtils.BINARY_MARKER, Messages.formatString(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR_FINDING_STATEMENT_CHANNEL), e);
}
}
long revision = -1;
BinaryStoreFile bsf = null;
ClientGraph graph = null;
File target = null;
File lockFile = null;
try {
if (update) {
graph = ac.getServerGraph(updateURI);
revision = graph.getRevision();
if (!isRevisioned(graph))
revision = -1;
//increase to the new revision;
if (revision != -1)
++revision;
bsf = getFileFromURI(updateURI, revision);
target = new File(bsf.getFilename());
lockFile = getLock(target);
if (lockFile == null) {
graph.close();
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_FILE_LOCKED, target.getAbsolutePath());
}
}
FileItem file = null;
try {
List<FileItem> items = upload.parseRequest(req);
for (FileItem item : items) {
if (!item.getFieldName().equals(FILENAME))
continue;
file = item;
break;
}
} catch (FileUploadException e) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_FILE_UPLOAD_ERROR, e);
} catch (Exception e) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_FILE_UPLOAD_ERROR, e);
}
if (file == null || StringUtils.isEmpty(file.getName())) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_NO_FILE_SENT);
}
if (scg != null) {
Map<String, Object> messageProperties = new HashMap<String, Object>();
Collection<Statement> statements = new HashSet<Statement>();
statements.add(Constants.valueFactory.createStatement(feedbackidURIg, BINARYSTORE_ITEM_PROGRESS_JOB_URI, BINARYSTORE_ITEM_CHECKSUM_JOB_URI, feedbackidURIg));
statements.add(Constants.valueFactory.createStatement(feedbackidURIg, BINARYSTORE_ITEM_PROGRESS_JOB_COMPLETE_URI, Constants.valueFactory.createLiteral(1), feedbackidURIg));
statements.add(Constants.valueFactory.createStatement(feedbackidURIg, BINARYSTORE_ITEM_PROGRESS_JOB_COMPLETED_URI, Constants.valueFactory.createLiteral(0), feedbackidURIg));
scg.sendMessage(messageProperties, statements);
}
String sha1sum = null;
try {
sha1sum = createChecksum(file.getInputStream());
} catch (IOException ioe) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR_CREATING_SHA1, file.getName());
}
if (scg != null) {
Map<String, Object> messageProperties = new HashMap<String, Object>();
Collection<Statement> statements = new HashSet<Statement>();
statements.add(Constants.valueFactory.createStatement(feedbackidURIg, BINARYSTORE_ITEM_PROGRESS_JOB_URI, BINARYSTORE_ITEM_CHECKSUM_JOB_URI, feedbackidURIg));
statements.add(Constants.valueFactory.createStatement(feedbackidURIg, BINARYSTORE_ITEM_PROGRESS_JOB_COMPLETE_URI, Constants.valueFactory.createLiteral(1), feedbackidURIg));
statements.add(Constants.valueFactory.createStatement(feedbackidURIg, BINARYSTORE_ITEM_PROGRESS_JOB_COMPLETED_URI, Constants.valueFactory.createLiteral(1), feedbackidURIg));
scg.sendMessage(messageProperties, statements);
}
String filename = new File(file.getName()).getName();
String contentType = file.getContentType();
long sizeInBytes = file.getSize();
if (update) {
Collection<Statement> statements = graph.find(updateURI, BINARYSTORE_ITEM_SHA_1_URI, null);
if (statements.size() > 0) {
Statement s = statements.iterator().next();
String sum = ((Literal) s.getObject()).getLabel();
if (sum.equals(sha1sum)) {
// no need to make a new revision as file is identical to currently stored file.
if (revision != -1)
--revision;
bsf = getFileFromURI(updateURI, revision);
sendSuccessMsg(req, resp, bsf.getURI(), revision);
return;
}
}
} else {
revision = (req.getParameter(REVISIONED).equals("true")) ? 0 : -1;
bsf = generateFilename(filename, revision);
target = new File(bsf.getFilename());
//create the parent Directories
File parentDir = target.getParentFile();
if (parentDir != null)
parentDir.mkdirs();
}
try {
//its a new file if it is a create operation or the binary file is revisioned.
if ((!update || (update && revision != -1)) && target.createNewFile() == false) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_FILE_ALREADY_EXISTS, target.getAbsolutePath());
}
} catch (IOException ioe) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR_CREATING_FILE, target.getAbsolutePath());
}
Resource fileURI = Constants.valueFactory.createResource(bsf.getURI().toString());
ac.begin();
if (!update) {
if (revision == -1) {
graph = ac.getServerGraph(bsf.getURI(), AnzoClient.NON_REVISIONED_NAMED_GRAPH);
} else {
graph = ac.getServerGraph(bsf.getURI(), AnzoClient.REVISIONED_NAMED_GRAPH);
}
graph.add(fileURI, RDF.TYPE, BINARYSTORE_ITEM_URI);
} else {
Collection<Statement> statements = graph.getStatements();
for (Statement s : statements) {
if (s.getObject() != BINARYSTORE_ITEM_URI) {
graph.remove(s);
}
}
}
//TODO: at this stage call out to any plugins which might want to add meta data to the graph.
graph.add(fileURI, BINARYSTORE_ITEM_SIZE_URI, Constants.valueFactory.createLiteral(sizeInBytes));
graph.add(fileURI, CONTENT_TYPE_URI, Constants.valueFactory.createLiteral(contentType));
graph.add(fileURI, DC.TITLE, Constants.valueFactory.createLiteral(filename));
graph.add(fileURI, BINARYSTORE_ITEM_SHA_1_URI, Constants.valueFactory.createLiteral(sha1sum));
// copy the file to the correct place on the server.
try {
copy(file.getInputStream(), new FileOutputStream(target), scg);
} catch (IOException ioe) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR_COPYING_FILE, file.getName(), target.getAbsolutePath());
}
ac.commit();
ac.updateRepository();
} finally {
if (graph != null)
graph.close();
if (lockFile != null)
lockFile.delete();
}
if (scg != null) {
scg.close();
try {
//sleep so that the feedback events have enough time to send the final bytes complete event.
Thread.sleep(200);
} catch (InterruptedException e) {
}
}
sendSuccessMsg(req, resp, bsf.getURI(), revision);
}
private String createFeedbackURI(AnzoClient ac) {
String user = ac.getServiceUser();
return BINARYSTORE_ITEM_PROGRESS_CHANNEL_PREFIX + user;
}
private static boolean isRevisioned(ClientGraph graph) {
INamedGraph metadata = graph.getMetadataGraph();
if (metadata.contains(graph.getNamedGraphUri(), NamedGraph.revisionedProperty, Constants.valueFactory.createLiteral(true))) {
return true;
}
return false;
}
private int read(AnzoClient ac, String url, HttpServletRequest request, HttpServletResponse response) throws ServletException, AnzoException {
IAnzoGraph graph = null;
try {
URI uri = Constants.valueFactory.createURI(url);
if (!ac.namedGraphExists(uri)) {
return HttpServletResponse.SC_NOT_FOUND;
}
if (!ac.canReadNamedGraph(uri)) {
response.setHeader(AUTH_HEADER, String.valueOf(HttpServletResponse.SC_UNAUTHORIZED));
return HttpServletResponse.SC_UNAUTHORIZED;
}
String requested_revision_str = request.getParameter(URL_QUERY_REVISION);
long requested_revision = -1;
if (requested_revision_str != null) {
requested_revision = Long.parseLong(requested_revision_str);
}
long revision = -1;
if (requested_revision == -1) {
ClientGraph g = ac.getServerGraph(uri);
revision = g.getRevision();
if (!isRevisioned(g))
revision = -1;
graph = g;
} else {
IAnzoGraph g = ac.getNamedGraphRevision(uri, requested_revision);
if (g != null && g.size() > 0) // An empty graph (and with empty metadatagraph) is returned for revisions that don't exist
revision = requested_revision;
else
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_FILE_REVISION_DOES_NOT_EXIST, Long.toString(requested_revision), uri.toString());
graph = g;
}
Boolean showMetaData = false;
if (request.getParameter(URL_QUERY_ASPECT) != null && request.getParameter(URL_QUERY_ASPECT).equals(URL_ASPECT_METADATA)) {
showMetaData = true;
}
//
if (showMetaData) {
String formatString = request.getParameter(URL_QUERY_FORMAT);
formatString = (formatString != null) ? formatString : RDF_XMLFORMAT;
try {
RDFFormat format = RDFFormat.forFileName("." + formatString);
if (format == null) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_UNRECOGNIZEDRDFFORMAT, formatString);
}
StringWriter sw = new StringWriter();
ReadWriteUtils.writeGraph(graph, sw);
response.setContentType(format.getDefaultMIMEType());
response.getOutputStream().print(sw.getBuffer().toString());
response.getOutputStream().flush();
return HttpServletResponse.SC_OK;
} catch (Exception e) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_ERROR, e);
}
}
BinaryStoreFile bsf = getFileFromURI(uri, revision);
String filename = bsf.getFilename();
Collection<Statement> cos = graph.find(uri, CONTENT_TYPE_URI, null);
if (cos.size() != 1) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_NO_CONTENTTYPE_FOUND, uri.toString());
}
Statement s = cos.iterator().next();
String contentType = (String) StatementUtils.getNativeValue((Literal) s.getObject());
//TODO: at this stage run plugins to do transformations on the file.
return serveStaticResource(request, response, filename, contentType);
} finally {
if (graph != null)
graph.close();
}
}
private void delete(AnzoClient ac, HttpServletRequest req, HttpServletResponse resp) throws AnzoException, ServletException {
String graphURI = req.getParameter(GRAPH);
if (graphURI == null) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_NO_GRAPHURI_SPECIFIED);
}
URI uri = Constants.valueFactory.createURI(graphURI);
if (!ac.namedGraphExists(uri))
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_NO_GRAPHURI_SPECIFIED);
if (!ac.canRemoveFromNamedGraph(uri))
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_UPDATE_PERMISSION_DENIED, uri.toString());
ClientGraph graph = ac.getServerGraph(uri);
if (!isRevisioned(graph)) {
// if it is a non revisioned file then delete the file else just delete the graph
if (!deleteFile(uri, -1))
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_FILE_LOCKED, uri.toString());
}
graph.getMetadataGraph().remove(uri, RDF.TYPE, NamedGraph.TYPE);
ac.updateRepository();
}
private boolean deleteFile(URI uri, long revision) {
BinaryStoreFile bsf = getFileFromURI(uri, revision);
File target = new File(bsf.getFilename());
File lockFile = getLock(target);
if (lockFile == null)
return false;
if (target.exists()) {
boolean success = target.delete();
lockFile.delete();
File parent = target;
while (success && !parent.getParentFile().getAbsolutePath().equals(serverNodeRootPath) && parent.getParentFile().listFiles().length == 0) {
parent = parent.getParentFile();
success = parent.delete();
}
}
return true;
}
private BinaryStoreFile generateFilename(String filename, long revision) {
filename = removeIllegalURIChars(filename);
Calendar cal = Calendar.getInstance();
int year = cal.get(Calendar.YEAR);
int month = (cal.get(Calendar.MONTH) + 1);
int day = cal.get(Calendar.DAY_OF_MONTH);
int dirInt = 0;
synchronized (directoryIncrementer) {
if (day != currentCal.get(Calendar.DAY_OF_MONTH) && month != (currentCal.get(Calendar.MONTH) + 1) && year != currentCal.get(Calendar.YEAR)) {
directoryIncrementer = 0;
}
currentCal = cal;
dirInt = ++directoryIncrementer;
}
String fn = serverNodeRootPath + File.separator;
fn += year + File.separator;
fn += month + File.separator;
fn += day + File.separator;
fn += dirInt + File.separator;
fn += filename;
fn += (revision == -1) ? "" : "-0";
String url = servletPath + "/";
url += serverNode + "/";
url += year + "/";
url += month + "/";
url += day + "/";
url += dirInt + "/";
url += filename;
return new BinaryStoreFile(fn, url);
//eg "/binarystore/" + servernode + "/" + date_year + "/" + date_month + "/" + date_day + "/" + incremented_integer + "/" + filename
}
private BinaryStoreFile getFileFromURI(URI u, long revision) {
String uri = u.toString();
if (!uri.startsWith(servletPath))
return null;
String file = serverRootPath + uri.substring(servletPath.length());
file += (revision != -1) ? "-" + revision : "";
File f = new File(file);
return new BinaryStoreFile(f.getAbsolutePath(), uri);
}
@SuppressWarnings("unchecked")
private int serveStaticResource(HttpServletRequest request, HttpServletResponse response, String filename, String contentType) throws AnzoException, ServletException {
// Find the resource and content
HttpContent content = null;
org.eclipse.jetty.util.resource.Resource resource = null;
try {
resource = org.eclipse.jetty.util.resource.Resource.newResource("file://" + filename);
Enumeration reqRanges = null;
// Is this a range request?
reqRanges = request.getHeaders(HttpHeaders.RANGE);
if (reqRanges != null && !reqRanges.hasMoreElements())
reqRanges = null;
// Handle resource
if (resource == null || !resource.exists())
return HttpServletResponse.SC_NOT_FOUND;
if (!resource.isDirectory()) {
// ensure we have content
content = new UnCachedContent(resource, contentType);
if (passConditionalHeaders(request, response, resource, content)) {
sendData(request, response, false, resource, content, reqRanges);
}
} else {
return HttpServletResponse.SC_NOT_FOUND;
}
} catch (IllegalArgumentException e) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_SENDING_DATA_FAILED, e, filename);
} catch (IOException e) {
throw new AnzoException(ExceptionConstants.BINARYSTORE.BINARYSTORE_SENDING_DATA_FAILED, e, filename);
} finally {
if (content != null) {
content.release();
}
if (resource != null) {
resource.release();
}
}
return HttpServletResponse.SC_OK;
}
Properties getInitProperties() {
Properties properties = new Properties();
@SuppressWarnings("unchecked")
// javax.servlet.GenericServlet returns unchecked value
Enumeration e = getInitParameterNames();
String paramName;
String paramValue;
while (e.hasMoreElements()) {
paramName = (String) e.nextElement();
paramValue = getInitParameter(paramName);
properties.put(paramName, paramValue);
}
return properties;
}
static private void sendNOOPResponse(HttpServletRequest req, HttpServletResponse resp) throws IOException {
boolean addHTML = false;
if ("application/json".equals(req.getHeader("Accept"))) {
resp.setContentType("application/json");
} else {
addHTML = true;
resp.setContentType("text/html");
}
resp.setStatus(HttpServletResponse.SC_OK);
PrintWriter out = resp.getWriter();
String response = "{ \"error\": false }";
out.print(addHTML ? envelopeHTMLMessage(response) : response);
out.flush();
out.close();
}
static private void sendJSONError(HttpServletRequest req, HttpServletResponse resp, AnzoException exception) throws IOException, JSONException {
boolean addHTML = false;
if ("application/json".equals(req.getHeader("Accept"))) {
resp.setContentType("application/json");
} else {
addHTML = true;
resp.setContentType("text/html");
}
resp.setStatus(HttpServletResponse.SC_OK);
PrintWriter out = resp.getWriter();
String jsonError = createJSONError(exception);
out.print(addHTML ? envelopeHTMLMessage(jsonError) : jsonError);
out.flush();
out.close();
}
static private String createJSONError(AnzoException exception) throws JSONException {
StringWriter sw = new StringWriter();
SerializationUtils.writeExceptionJSON(exception, sw);
return sw.toString();
}
static private void sendSuccessMsg(HttpServletRequest req, HttpServletResponse resp, URI uri, long revision) throws IOException, JSONException {
boolean addHTML = false;
if ("application/json".equals(req.getHeader("Accept"))) {
resp.setContentType("application/json");
} else {
addHTML = true;
resp.setContentType("text/html");
}
resp.setStatus(HttpServletResponse.SC_OK);
PrintWriter out = resp.getWriter();
out.print(addHTML ? envelopeHTMLMessage(createReturnMessage(uri, revision)) : createReturnMessage(uri, revision));
out.flush();
out.close();
}
static private String createReturnMessage(URI uri, long revision) throws JSONException {
StringWriter sw = new StringWriter();
JSONWriter jj = new JSONWriter(sw);
jj.object();
jj.key("error");
jj.value(false);
jj.key("uri");
jj.value(uri.toString());
jj.key("revision");
jj.value(revision);
jj.endObject();
return sw.toString();
}
static private String envelopeHTMLMessage(String str) {
return ("<html> <head></head><body> <textarea style='width: 100%; height: 100%;'>" + str + "</textarea></body></html>");
}
static void copy(InputStream in, OutputStream out, IStatementChannel scg) throws IOException {
// Do not allow other threads to read from the input
// or write to the output while copying is taking place
synchronized (in) {
synchronized (out) {
byte[] buffer = new byte[256];
while (true) {
int bytesRead = in.read(buffer);
if (bytesRead == -1)
break;
out.write(buffer, 0, bytesRead);
}
}
}
}
static String removeIllegalURIChars(String s) {
char[] chars = s.toCharArray();
StringBuilder b = new StringBuilder();
for (char c : chars) {
if (c == '\'' || c == '"' || c == ' ')
b.append("_");
else
b.append(c);
}
return b.toString();
}
private File getLock(File target) {
File parent = target.getParentFile();
File lockfile = new File(parent, nodelockid);
boolean exists = true;
try {
exists = lockfile.createNewFile();
} catch (IOException e) {
return null;
}
if (!exists) {
return null;
}
File[] files = parent.listFiles();
for (File f : files) {
if (!f.equals(lockfile)) {
if (f.getName().startsWith(LOCKFILE_PREFIX)) {
String lock = f.getName().substring(LOCKFILE_PREFIX.length());
String[] nodeid = lock.split(LOCKFILE_DELIMETER);
if (nodeid.length != 2)
continue;
boolean validLock = isLockValid(nodeid[0], nodeid[1]);
if (!validLock) {
//its an invalid lockfile - presumably created by a crashed or shutdown server so lets clean it up.
f.delete();
} else {
lockfile.delete();
return null;
}
}
}
}
return lockfile;
}
private boolean isLockValid(String node, String id) {
File nodeDir = new File(varDirFile, node);
if (nodeDir.exists()) {
File idFile = new File(nodeDir, id);
if (idFile.exists()) {
Date d = new Date();
if (d.getTime() < idFile.lastModified() + BINARYSTORE_HEARTBEAT_CHECKTIME) {
return true;
} else {
//clean up the file as it is an old one.
idFile.delete();
}
}
}
return false;
}
static String createChecksum(InputStream fis) throws IOException {
byte[] buffer = new byte[1024];
MessageDigest complete = null;
try {
complete = MessageDigest.getInstance("SHA-1");
} catch (NoSuchAlgorithmException e) {
log.warn(LogUtils.SERVER_INTERNAL_MARKER, Messages.formatString(ExceptionConstants.BINARYSTORE.BINARYSTORE_NO_SHA1));
return null;
}
int numRead;
do {
numRead = fis.read(buffer);
if (numRead > 0) {
complete.update(buffer, 0, numRead);
}
} while (numRead != -1);
fis.close();
byte[] digest = complete.digest();
String hash = "";
for (int i = 0; i < digest.length; i++) {
String hex = Integer.toHexString(digest[i]);
if (hex.length() == 1)
hex = "0" + hex;
hex = hex.substring(hex.length() - 2);
hash += hex;
}
return hash;
}
private void deleteDirectory(File file) {
if (file.isDirectory()) {
for (File subFile : file.listFiles()) {
deleteDirectory(subFile);
}
}
file.delete();
}
class BinaryStoreFile {
private String _filename = null;
private URI _uri = null;
public BinaryStoreFile(String filename, String url) {
_filename = filename;
_uri = Constants.valueFactory.createURI(url);
}
public String getFilename() {
return _filename;
}
public URI getURI() {
return _uri;
}
}
}