/*
* Copyright (c) 2017, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
*
* WSO2 Inc. licenses this file to you under the Apache License,
* Version 2.0 (the "License"); you may not use this file except
* in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.wso2.carbon.transport.file.connector.server;
import org.apache.commons.vfs2.FileContent;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.FileSystemManager;
import org.apache.commons.vfs2.FileSystemOptions;
import org.apache.commons.vfs2.FileType;
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
import org.apache.commons.vfs2.provider.UriParser;
import org.apache.commons.vfs2.provider.ftp.FtpFileSystemConfigBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wso2.carbon.messaging.CarbonMessage;
import org.wso2.carbon.messaging.CarbonMessageProcessor;
import org.wso2.carbon.messaging.StreamingCarbonMessage;
import org.wso2.carbon.messaging.exceptions.ServerConnectorException;
import org.wso2.carbon.transport.file.connector.server.exception.FileServerConnectorException;
import org.wso2.carbon.transport.file.connector.server.util.Constants;
import org.wso2.carbon.transport.file.connector.server.util.FileTransportUtils;
import java.io.InputStream;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
/**
* Provides the capability to process a file and delete it afterwards.
*/
public class FileConsumer {
private static final Logger log = LoggerFactory.getLogger(FileConsumer.class);
private Map<String, String> fileProperties;
private FileSystemManager fsManager = null;
private String serviceName;
private CarbonMessageProcessor messageProcessor;
private String fileURI;
private FileObject fileObject;
private FileSystemOptions fso;
/**
* Time-out interval (in mill-seconds) to wait for the callback.
*/
private long timeOutInterval = 30000;
private boolean deleteIfNotAck = false;
public FileConsumer(String id, Map<String, String> fileProperties,
CarbonMessageProcessor messageProcessor)
throws ServerConnectorException {
this.serviceName = id;
this.fileProperties = fileProperties;
this.messageProcessor = messageProcessor;
setupParams();
try {
StandardFileSystemManager fsm = new StandardFileSystemManager();
fsm.setConfiguration(getClass().getClassLoader().getResource("providers.xml"));
fsm.init();
fsManager = fsm;
} catch (FileSystemException e) {
throw new ServerConnectorException("Could not initialize File System Manager from " +
"the configuration: providers.xml", e);
}
Map<String, String> options = parseSchemeFileOptions(fileURI);
fso = FileTransportUtils.attachFileSystemOptions(options, fsManager);
if (options != null && Constants.SCHEME_FTP.equals(options.get(Constants.SCHEME))) {
FtpFileSystemConfigBuilder.getInstance().setPassiveMode(fso, true);
}
try {
fileObject = fsManager.resolveFile(fileURI, fso);
} catch (FileSystemException e) {
throw new FileServerConnectorException("Failed to resolve fileURI: "
+ FileTransportUtils.maskURLPassword(fileURI), e);
}
}
/**
* Do the file processing operation for the given set of properties. Do the
* checks and pass the control to processFile method
*/
public void consume() throws FileServerConnectorException {
if (log.isDebugEnabled()) {
log.debug("Polling for directory or file : " +
FileTransportUtils.maskURLPassword(fileURI));
}
// If file/folder found proceed to the processing stage
try {
boolean isFileExists;
try {
isFileExists = fileObject.exists();
} catch (FileSystemException e) {
throw new FileServerConnectorException("Error occurred when determining whether the file at URI : "
+ FileTransportUtils.maskURLPassword(fileURI) + " exists. " + e);
}
boolean isFileReadable;
try {
isFileReadable = fileObject.isReadable();
} catch (FileSystemException e) {
throw new FileServerConnectorException("Error occurred when determining whether the file at URI : "
+ FileTransportUtils.maskURLPassword(fileURI) + " is readable. " + e);
}
if (isFileExists && isFileReadable) {
FileType fileType;
try {
fileType = fileObject.getType();
} catch (FileSystemException e) {
throw new FileServerConnectorException("Error occurred when determining whether file: "
+ FileTransportUtils.maskURLPassword(fileURI) + " is a file or a folder", e);
}
if (fileType == FileType.FILE) {
processFile(fileObject);
deleteFile(fileObject);
} else if (fileType == FileType.FOLDER) {
FileObject[] children = null;
try {
children = fileObject.getChildren();
} catch (FileSystemException ignored) {
if (log.isDebugEnabled()) {
log.debug("The file does not exist, or is not a folder, or an error " +
"has occurred when trying to list the children. File URI : "
+ FileTransportUtils.maskURLPassword(fileURI), ignored);
}
}
// if this is a file that would translate to a single message
if (children == null || children.length == 0) {
if (log.isDebugEnabled()) {
log.debug("Folder at " + FileTransportUtils.maskURLPassword(fileURI)
+ " is empty.");
}
} else {
directoryHandler(children);
}
} else {
throw new FileServerConnectorException("File: "
+ FileTransportUtils.maskURLPassword(fileURI) + " is neither a file or " +
"a folder" + (fileType == null ? "" : ". Found file type: " + fileType.toString()));
}
} else {
throw new FileServerConnectorException("Unable to access or read file or directory : "
+ FileTransportUtils.maskURLPassword(fileURI)
+ ". Reason: "
+ (isFileExists ? (isFileReadable ? "Unknown reason"
: "The file can not be read!") : "The file does not exist!"));
}
} finally {
try {
fileObject.close();
} catch (FileSystemException e) {
log.warn("Could not close file at URI: " + FileTransportUtils.maskURLPassword(fileURI), e);
}
}
if (log.isDebugEnabled()) {
log.debug("End : Scanning directory or file : " + FileTransportUtils.maskURLPassword(fileURI));
}
}
/**
* Setup the required transport parameters
*/
private void setupParams() throws ServerConnectorException {
fileURI = fileProperties.get(Constants.TRANSPORT_FILE_FILE_URI);
if (fileURI == null) {
throw new ServerConnectorException(Constants.TRANSPORT_FILE_FILE_URI + " is a " +
"mandatory parameter for " + Constants.PROTOCOL_NAME + " transport.");
}
if (fileURI.trim().equals("")) {
throw new ServerConnectorException(Constants.TRANSPORT_FILE_FILE_URI + " parameter " +
"cannot be empty for " + Constants.PROTOCOL_NAME + " transport.");
}
String timeOut = fileProperties.get(Constants.FILE_ACKNOWLEDGEMENT_TIME_OUT);
if (timeOut != null) {
try {
timeOutInterval = Long.parseLong(timeOut);
} catch (NumberFormatException e) {
log.error("Provided " + Constants.FILE_ACKNOWLEDGEMENT_TIME_OUT + " is invalid. " +
"Using the default callback timeout, " +
timeOutInterval + " milliseconds", e);
}
}
String strDeleteIfNotAck = fileProperties.get(Constants.FILE_DELETE_IF_NOT_ACKNOWLEDGED);
if (strDeleteIfNotAck != null) {
deleteIfNotAck = Boolean.parseBoolean(strDeleteIfNotAck);
}
}
private Map<String, String> parseSchemeFileOptions(String fileURI) {
String scheme = UriParser.extractScheme(fileURI);
if (scheme == null) {
return null;
}
HashMap<String, String> schemeFileOptions = new HashMap<>();
schemeFileOptions.put(Constants.SCHEME, scheme);
addOptions(scheme, schemeFileOptions);
return schemeFileOptions;
}
private void addOptions(String scheme, Map<String, String> schemeFileOptions) {
if (scheme.equals(Constants.SCHEME_SFTP)) {
for (Constants.SftpFileOption option : Constants.SftpFileOption.values()) {
String strValue = fileProperties.get(Constants.SFTP_PREFIX + option.toString());
if (strValue != null && !strValue.equals("")) {
schemeFileOptions.put(option.toString(), strValue);
}
}
}
}
/**
* Handle directory with chile elements
*
* @param children
* @return
* @throws FileSystemException
*/
private void directoryHandler(FileObject[] children) throws FileServerConnectorException {
// Sort the files
String strSortParam = fileProperties.get(Constants.FILE_SORT_PARAM);
if (strSortParam != null && !"NONE".equals(strSortParam)) {
log.debug("Starting to sort the files in folder: " + FileTransportUtils.maskURLPassword(fileURI));
String strSortOrder = fileProperties.get(Constants.FILE_SORT_ORDER);
boolean bSortOrderAscending = true;
if (strSortOrder != null) {
bSortOrderAscending = Boolean.parseBoolean(strSortOrder);
}
if (log.isDebugEnabled()) {
log.debug("Sorting the files by : " + strSortOrder + ". (" +
bSortOrderAscending + ")");
}
switch (strSortParam) {
case Constants.FILE_SORT_VALUE_NAME:
if (bSortOrderAscending) {
Arrays.sort(children, new FileNameAscComparator());
} else {
Arrays.sort(children, new FileNameDesComparator());
}
break;
case Constants.FILE_SORT_VALUE_SIZE:
if (bSortOrderAscending) {
Arrays.sort(children, new FileSizeAscComparator());
} else {
Arrays.sort(children, new FileSizeDesComparator());
}
break;
case Constants.FILE_SORT_VALUE_LASTMODIFIEDTIMESTAMP:
if (bSortOrderAscending) {
Arrays.sort(children, new FileLastmodifiedtimestampAscComparator());
} else {
Arrays.sort(children, new FileLastmodifiedtimestampDesComparator());
}
break;
default:
log.warn("Invalid value given for " + Constants.FILE_SORT_PARAM + " parameter. "
+ " Expected one of the values: " + Constants.FILE_SORT_VALUE_NAME + ", "
+ Constants.FILE_SORT_VALUE_SIZE + " or "
+ Constants.FILE_SORT_VALUE_LASTMODIFIEDTIMESTAMP + ". Found: " + strSortParam);
break;
}
if (log.isDebugEnabled()) {
log.debug("End sorting the files.");
}
}
for (FileObject child : children) {
processFile(child);
deleteFile(child);
//close the file system after processing
try {
child.close();
} catch (FileSystemException e) {
log.warn("Could not close the file: " + child.getName().getPath(), e);
}
}
}
/**
* Actual processing of the file/folder
*
* @param file
* @return
*/
private FileObject processFile(FileObject file) throws FileServerConnectorException {
FileContent content;
String fileURI;
String fileName = file.getName().getBaseName();
String filePath = file.getName().getPath();
fileURI = file.getName().getURI();
try {
content = file.getContent();
} catch (FileSystemException e) {
throw new FileServerConnectorException("Could not read content of file at URI: "
+ FileTransportUtils.maskURLPassword(fileURI) + ". ", e);
}
InputStream inputStream;
try {
inputStream = content.getInputStream();
} catch (FileSystemException e) {
throw new FileServerConnectorException("Error occurred when trying to get " +
"input stream from file at URI :" + FileTransportUtils.maskURLPassword(fileURI), e);
}
CarbonMessage cMessage = new StreamingCarbonMessage(inputStream);
cMessage.setProperty(org.wso2.carbon.messaging.Constants.PROTOCOL, Constants.PROTOCOL_NAME);
cMessage.setProperty(Constants.FILE_TRANSPORT_PROPERTY_SERVICE_NAME, serviceName);
cMessage.setHeader(Constants.FILE_PATH, filePath);
cMessage.setHeader(Constants.FILE_NAME, fileName);
cMessage.setHeader(Constants.FILE_URI, fileURI);
try {
cMessage.setHeader(Constants.FILE_LENGTH, Long.toString(content.getSize()));
cMessage.setHeader(Constants.LAST_MODIFIED, Long.toString(content.getLastModifiedTime()));
} catch (FileSystemException e) {
log.warn("Unable to set file length or last modified date header.", e);
}
FileServerConnectorCallback callback = new FileServerConnectorCallback();
try {
messageProcessor.receive(cMessage, callback);
} catch (Exception e) {
throw new FileServerConnectorException("Failed to send stream from file: "
+ FileTransportUtils.maskURLPassword(fileURI) + " to message processor. ", e);
}
try {
callback.waitTillDone(timeOutInterval, deleteIfNotAck, fileURI);
} catch (InterruptedException e) {
throw new FileServerConnectorException("Interrupted while waiting for message " +
"processor to consume the file input stream. Aborting processing of file: " +
FileTransportUtils.maskURLPassword(fileURI), e);
}
return file;
}
/**
* Do the post processing actions
*
* @param fileObject
*/
private void deleteFile(FileObject fileObject) throws FileServerConnectorException {
if (log.isDebugEnabled()) {
log.debug("Deleting file :" + FileTransportUtils.
maskURLPassword(fileObject.getName().getBaseName()));
}
try {
if (!fileObject.delete()) {
throw new FileServerConnectorException("Could not delete file : "
+ FileTransportUtils.maskURLPassword(fileObject.getName().getBaseName()));
}
} catch (FileSystemException e) {
throw new FileServerConnectorException("Could not delete file : "
+ FileTransportUtils.maskURLPassword(fileObject.getName().getBaseName()), e);
}
}
/**
* Comparator classed used to sort the files according to user input
*/
static class FileNameAscComparator implements Comparator<FileObject>, Serializable {
private static final long serialVersionUID = 1;
@Override
public int compare(FileObject o1, FileObject o2) {
return o1.getName().compareTo(o2.getName());
}
}
static class FileLastmodifiedtimestampAscComparator
implements Comparator<FileObject>, Serializable {
private static final long serialVersionUID = 1;
@Override
public int compare(FileObject o1, FileObject o2) {
Long lDiff = 0L;
try {
lDiff = o1.getContent().getLastModifiedTime()
- o2.getContent().getLastModifiedTime();
} catch (FileSystemException e) {
log.warn("Unable to compare last modified timestamp of the two files.", e);
}
return lDiff.intValue();
}
}
static class FileSizeAscComparator implements Comparator<FileObject>, Serializable {
private static final long serialVersionUID = 1;
@Override
public int compare(FileObject o1, FileObject o2) {
Long lDiff = 0L;
try {
lDiff = o1.getContent().getSize() - o2.getContent().getSize();
} catch (FileSystemException e) {
log.warn("Unable to compare size of the two files.", e);
}
return lDiff.intValue();
}
}
static class FileNameDesComparator implements Comparator<FileObject>, Serializable {
private static final long serialVersionUID = 1;
@Override
public int compare(FileObject o1, FileObject o2) {
return o2.getName().compareTo(o1.getName());
}
}
static class FileLastmodifiedtimestampDesComparator
implements Comparator<FileObject>, Serializable {
private static final long serialVersionUID = 1;
@Override
public int compare(FileObject o1, FileObject o2) {
Long lDiff = 0L;
try {
lDiff = o2.getContent().getLastModifiedTime()
- o1.getContent().getLastModifiedTime();
} catch (FileSystemException e) {
log.warn("Unable to compare last modified timestamp of the two files.", e);
}
return lDiff.intValue();
}
}
static class FileSizeDesComparator implements Comparator<FileObject>, Serializable {
private static final long serialVersionUID = 1;
@Override
public int compare(FileObject o1, FileObject o2) {
Long lDiff = 0L;
try {
lDiff = o2.getContent().getSize() - o1.getContent().getSize();
} catch (FileSystemException e) {
log.warn("Unable to compare size of the two files.", e);
}
return lDiff.intValue();
}
}
}