/**
* Licensed to JumpMind Inc under one or more contributor
* license agreements. See the NOTICE file distributed
* with this work for additional information regarding
* copyright ownership. JumpMind Inc licenses this file
* to you under the GNU General Public License, version 3.0 (GPLv3)
* (the "License"); you may not use this file except in compliance
* with the License.
*
* You should have received a copy of the GNU General Public License,
* version 3.0 (GPLv3) along with this library; if not, see
* <http://www.gnu.org/licenses/>.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jumpmind.symmetric.io;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSystemException;
import org.apache.commons.vfs.FileSystemOptions;
import org.apache.commons.vfs.Selectors;
import org.apache.commons.vfs.impl.StandardFileSystemManager;
import org.apache.commons.vfs.provider.ftp.FtpFileSystemConfigBuilder;
import org.apache.commons.vfs.provider.sftp.SftpFileSystemConfigBuilder;
import org.jumpmind.db.model.Table;
import org.jumpmind.exception.IoException;
import org.jumpmind.symmetric.io.data.Batch;
import org.jumpmind.symmetric.io.data.CsvData;
import org.jumpmind.symmetric.io.data.DataContext;
import org.jumpmind.symmetric.io.data.DataEventType;
import org.jumpmind.symmetric.io.data.IDataWriter;
import org.jumpmind.symmetric.io.data.writer.DataWriterStatisticConstants;
import org.jumpmind.util.Statistics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FtpDataWriter implements IDataWriter {
public enum Format {
CSV
}
public enum Protocol {
FTP, SFTP
}
protected static final Logger logger = LoggerFactory.getLogger(FtpDataWriter.class);
protected String server;
protected String username;
protected String password;
protected FtpDataWriter.Protocol protocol = Protocol.FTP;
protected FtpDataWriter.Format format = Format.CSV;
protected String stagingDir;
protected String remoteDir;
protected Batch batch;
protected Table table;
protected Map<String, FileInfo> fileInfoByTable = new HashMap<String, FtpDataWriter.FileInfo>();
protected StandardFileSystemManager manager = new StandardFileSystemManager();
protected Map<Batch, Statistics> statistics = new HashMap<Batch, Statistics>();
public void setRemoteDir(String remoteDir) {
this.remoteDir = remoteDir;
}
public void setFormat(FtpDataWriter.Format format) {
this.format = format;
}
public void setPassword(String password) {
this.password = password;
}
public void setServer(String server) {
this.server = server;
}
public void setProtocol(FtpDataWriter.Protocol protocol) {
this.protocol = protocol;
}
public void setUsername(String username) {
this.username = username;
}
public void setStagingDir(String stagingDir) {
this.stagingDir = stagingDir;
}
public void open(DataContext context) {
try {
manager.init();
} catch (FileSystemException e) {
throw new IoException(e);
}
}
public void close() {
}
public Map<Batch, Statistics> getStatistics() {
return statistics;
}
public void start(Batch batch) {
this.batch = batch;
this.statistics.put(batch, new Statistics());
}
public boolean start(Table table) {
this.table = table;
createFile(table, batch);
return true;
}
public void write(CsvData data) {
if (format == Format.CSV
&& (data.getDataEventType() == DataEventType.UPDATE || data.getDataEventType() == DataEventType.INSERT)) {
printCsvData(data);
}
}
public void end(Table table) {
}
public void end(Batch batch, boolean inError) {
if (!inError) {
try {
closeFiles();
sendFiles();
} finally {
deleteFiles();
}
} else {
try {
closeFiles();
} finally {
deleteFiles();
}
}
}
protected void createFile(Table table, Batch batch) {
FileInfo fileInfo = fileInfoByTable.get(table.getFullyQualifiedTableName());
if (fileInfo == null) {
try {
fileInfo = new FileInfo();
fileInfo.outputFile = new File(stagingDir, batch.getBatchId() + "-"
+ table.getFullyQualifiedTableName() + "." + format.name().toLowerCase());
fileInfo.outputFile.getParentFile().mkdirs();
fileInfo.outputFileWriter = new BufferedWriter(new FileWriter(fileInfo.outputFile));
fileInfoByTable.put(table.getFullyQualifiedTableName(), fileInfo);
if (format == Format.CSV) {
printCsvTableHeader();
}
} catch (IOException e) {
throw new IoException(e);
}
}
}
protected void closeFiles() {
Collection<FileInfo> fileInfos = fileInfoByTable.values();
for (FileInfo fileInfo : fileInfos) {
try {
fileInfo.outputFileWriter.flush();
fileInfo.outputFileWriter.close();
} catch (IOException e) {
logger.warn("", e);
}
}
}
protected void deleteFiles() {
Collection<FileInfo> fileInfos = fileInfoByTable.values();
for (FileInfo fileInfo : fileInfos) {
try {
fileInfo.outputFileWriter.close();
} catch (IOException e) {
logger.warn("", e);
} finally {
FileUtils.deleteQuietly(fileInfo.outputFile);
}
}
fileInfoByTable.clear();
}
protected void sendFiles() {
if (fileInfoByTable.size() > 0) {
try {
String sftpUri = buildUri();
FileSystemOptions opts = new FileSystemOptions();
FtpFileSystemConfigBuilder.getInstance().setUserDirIsRoot(opts, true);
SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no");
SftpFileSystemConfigBuilder.getInstance().setTimeout(opts, 60000);
Collection<FileInfo> fileInfos = fileInfoByTable.values();
for (FileInfo fileInfo : fileInfos) {
FileObject fileObject = manager.resolveFile(
sftpUri + "/" + fileInfo.outputFile.getName(), opts);
FileObject localFileObject = manager.resolveFile(fileInfo.outputFile
.getAbsolutePath());
fileObject.copyFrom(localFileObject, Selectors.SELECT_SELF);
fileObject.close();
}
} catch (FileSystemException e) {
logger.warn("If you have not configured your ftp connection it should be configured in conf/ftp-extensions.xml");
throw new IoException(e);
} catch (Exception e) {
throw new IoException(e);
} finally {
manager.close();
}
}
}
protected String buildUri() {
String credentials = "";
if (StringUtils.isNotBlank(username)) {
credentials = username;
if (StringUtils.isNotBlank(password)) {
credentials = credentials + ":" + password;
}
credentials = credentials + "@";
}
String path = "";
if (StringUtils.isNotBlank(this.remoteDir)) {
path = "/" + this.remoteDir;
}
return protocol.toString().toLowerCase() + "://" + credentials + server + path;
}
protected void printCsvTableHeader() {
println(table.getColumnNames());
}
protected void printCsvData(CsvData data) {
println(data.getParsedData(CsvData.ROW_DATA));
}
protected void println(String... data) {
FileInfo fileInfo = fileInfoByTable.get(table.getFullyQualifiedTableName());
if (fileInfo != null) {
try {
StringBuilder buffer = new StringBuilder();
for (int i = 0; i < data.length; i++) {
if (i != 0) {
buffer.append(",");
}
buffer.append(data[i]);
}
buffer.append("\n");
fileInfo.outputFileWriter.write(buffer.toString());
long byteCount = buffer.length();
statistics.get(batch).increment(DataWriterStatisticConstants.BYTECOUNT, byteCount);
} catch (IOException e) {
throw new IoException(e);
}
}
}
class FileInfo {
File outputFile;
BufferedWriter outputFileWriter;
}
}