/*! ******************************************************************************
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/
package org.pentaho.di.job.entries.ssh2get;
import java.io.File;
import java.io.FileOutputStream;
import java.util.Iterator;
import java.util.List;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileType;
import org.pentaho.di.cluster.SlaveServer;
import org.pentaho.di.core.CheckResultInterface;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.Result;
import org.pentaho.di.core.annotations.JobEntry;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.encryption.Encr;
import org.pentaho.di.core.exception.KettleDatabaseException;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleXMLException;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.core.xml.XMLHandler;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.job.Job;
import org.pentaho.di.job.JobMeta;
import org.pentaho.di.job.entries.FTPUtils;
import org.pentaho.di.job.entry.JobEntryBase;
import org.pentaho.di.job.entry.JobEntryInterface;
import org.pentaho.di.job.entry.validator.AndValidator;
import org.pentaho.di.job.entry.validator.JobEntryValidatorUtils;
import org.pentaho.di.repository.ObjectId;
import org.pentaho.di.repository.Repository;
import org.pentaho.di.resource.ResourceEntry;
import org.pentaho.di.resource.ResourceEntry.ResourceType;
import org.pentaho.di.resource.ResourceReference;
import org.pentaho.metastore.api.IMetaStore;
import org.w3c.dom.Node;
import com.trilead.ssh2.Connection;
import com.trilead.ssh2.HTTPProxyData;
import com.trilead.ssh2.KnownHosts;
import com.trilead.ssh2.SFTPv3Client;
import com.trilead.ssh2.SFTPv3DirectoryEntry;
import com.trilead.ssh2.SFTPv3FileAttributes;
import com.trilead.ssh2.SFTPv3FileHandle;
/**
* This defines a SSH2 GET job entry.
*
* @author Samatar
* @since 17-12-2007
*
*/
@JobEntry(
id = "SSH2_GET",
image = "SHG.svg",
i18nPackageName = "org.pentaho.di.job.entries.ssh2get",
name = "JobSSH2GET.TypeDesc",
description = "JobSSH2GET.Tooltip",
categoryDescription = "i18n:org.pentaho.di.job:JobCategory.Category.Deprecated" )
public class JobEntrySSH2GET extends JobEntryBase implements Cloneable, JobEntryInterface {
private static Class<?> PKG = JobEntrySSH2GET.class; // for i18n purposes, needed by Translator2!!
private String serverName;
private String userName;
private String password;
private String serverPort;
private String ftpDirectory;
private String localDirectory;
private String wildcard;
private boolean onlyGettingNewFiles; /* Don't overwrite files */
private boolean usehttpproxy;
private String httpProxyHost;
private String httpproxyport;
private String httpproxyusername;
private String httpProxyPassword;
private boolean publicpublickey;
private String keyFilename;
private String keyFilePass;
private boolean useBasicAuthentication;
private String afterFtpPut;
private String destinationfolder;
private boolean createdestinationfolder;
private boolean cachehostkey;
private int timeout;
boolean createtargetfolder;
boolean includeSubFolders;
static KnownHosts database = new KnownHosts();
int nbfilestoget = 0;
int nbgot = 0;
int nbrerror = 0;
public JobEntrySSH2GET( String n ) {
super( n, "" );
serverName = null;
publicpublickey = false;
keyFilename = null;
keyFilePass = null;
usehttpproxy = false;
httpProxyHost = null;
httpproxyport = null;
httpproxyusername = null;
httpProxyPassword = null;
serverPort = "22";
useBasicAuthentication = false;
afterFtpPut = "do_nothing";
destinationfolder = null;
includeSubFolders = false;
createdestinationfolder = false;
createtargetfolder = false;
cachehostkey = false;
timeout = 0;
}
public JobEntrySSH2GET() {
this( "" );
}
@Override
public Object clone() {
JobEntrySSH2GET je = (JobEntrySSH2GET) super.clone();
return je;
}
@Override
public String getXML() {
StringBuffer retval = new StringBuffer( 128 );
retval.append( super.getXML() );
retval.append( " " ).append( XMLHandler.addTagValue( "servername", serverName ) );
retval.append( " " ).append( XMLHandler.addTagValue( "username", userName ) );
retval.append( " " ).append(
XMLHandler.addTagValue( "password", Encr.encryptPasswordIfNotUsingVariables( getPassword() ) ) );
retval.append( " " ).append( XMLHandler.addTagValue( "serverport", serverPort ) );
retval.append( " " ).append( XMLHandler.addTagValue( "ftpdirectory", ftpDirectory ) );
retval.append( " " ).append( XMLHandler.addTagValue( "localdirectory", localDirectory ) );
retval.append( " " ).append( XMLHandler.addTagValue( "wildcard", wildcard ) );
retval.append( " " ).append( XMLHandler.addTagValue( "only_new", onlyGettingNewFiles ) );
retval.append( " " ).append( XMLHandler.addTagValue( "usehttpproxy", usehttpproxy ) );
retval.append( " " ).append( XMLHandler.addTagValue( "httpproxyhost", httpProxyHost ) );
retval.append( " " ).append( XMLHandler.addTagValue( "httpproxyport", httpproxyport ) );
retval.append( " " ).append( XMLHandler.addTagValue( "httpproxyusername", httpproxyusername ) );
retval.append( " " ).append( XMLHandler.addTagValue( "httpproxypassword", httpProxyPassword ) );
retval.append( " " ).append( XMLHandler.addTagValue( "publicpublickey", publicpublickey ) );
retval.append( " " ).append( XMLHandler.addTagValue( "keyfilename", keyFilename ) );
retval.append( " " ).append( XMLHandler.addTagValue( "keyfilepass", keyFilePass ) );
retval.append( " " ).append( XMLHandler.addTagValue( "usebasicauthentication", useBasicAuthentication ) );
retval.append( " " ).append( XMLHandler.addTagValue( "afterftpput", afterFtpPut ) );
retval.append( " " ).append( XMLHandler.addTagValue( "destinationfolder", destinationfolder ) );
retval
.append( " " ).append( XMLHandler.addTagValue( "createdestinationfolder", createdestinationfolder ) );
retval.append( " " ).append( XMLHandler.addTagValue( "cachehostkey", cachehostkey ) );
retval.append( " " ).append( XMLHandler.addTagValue( "timeout", timeout ) );
retval.append( " " ).append( XMLHandler.addTagValue( "createtargetfolder", createtargetfolder ) );
retval.append( " " ).append( XMLHandler.addTagValue( "includeSubFolders", includeSubFolders ) );
return retval.toString();
}
@Override
public void loadXML( Node entrynode, List<DatabaseMeta> databases, List<SlaveServer> slaveServers,
Repository rep, IMetaStore metaStore ) throws KettleXMLException {
try {
super.loadXML( entrynode, databases, slaveServers );
serverName = XMLHandler.getTagValue( entrynode, "servername" );
userName = XMLHandler.getTagValue( entrynode, "username" );
password = Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( entrynode, "password" ) );
serverPort = XMLHandler.getTagValue( entrynode, "serverport" );
ftpDirectory = XMLHandler.getTagValue( entrynode, "ftpdirectory" );
localDirectory = XMLHandler.getTagValue( entrynode, "localdirectory" );
wildcard = XMLHandler.getTagValue( entrynode, "wildcard" );
onlyGettingNewFiles = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "only_new" ) );
usehttpproxy = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "usehttpproxy" ) );
httpProxyHost = XMLHandler.getTagValue( entrynode, "httpproxyhost" );
httpproxyport = XMLHandler.getTagValue( entrynode, "httpproxyport" );
httpproxyusername = XMLHandler.getTagValue( entrynode, "httpproxyusername" );
httpProxyPassword = XMLHandler.getTagValue( entrynode, "httpproxypassword" );
publicpublickey = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "publicpublickey" ) );
keyFilename = XMLHandler.getTagValue( entrynode, "keyfilename" );
keyFilePass = XMLHandler.getTagValue( entrynode, "keyfilepass" );
useBasicAuthentication =
"Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "usebasicauthentication" ) );
afterFtpPut = XMLHandler.getTagValue( entrynode, "afterftpput" );
destinationfolder = XMLHandler.getTagValue( entrynode, "destinationfolder" );
createdestinationfolder =
"Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "createdestinationfolder" ) );
cachehostkey = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "cachehostkey" ) );
timeout = Const.toInt( XMLHandler.getTagValue( entrynode, "timeout" ), 0 );
createtargetfolder = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "createtargetfolder" ) );
includeSubFolders = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "includeSubFolders" ) );
} catch ( KettleXMLException xe ) {
throw new KettleXMLException( BaseMessages.getString( PKG, "JobSSH2GET.Log.UnableLoadXML", xe.getMessage() ) );
}
}
@Override
public void loadRep( Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List<DatabaseMeta> databases,
List<SlaveServer> slaveServers ) throws KettleException {
try {
serverName = rep.getJobEntryAttributeString( id_jobentry, "servername" );
userName = rep.getJobEntryAttributeString( id_jobentry, "username" );
password =
Encr.decryptPasswordOptionallyEncrypted( rep.getJobEntryAttributeString( id_jobentry, "password" ) );
serverPort = rep.getJobEntryAttributeString( id_jobentry, "serverport" );
ftpDirectory = rep.getJobEntryAttributeString( id_jobentry, "ftpdirectory" );
localDirectory = rep.getJobEntryAttributeString( id_jobentry, "localdirectory" );
wildcard = rep.getJobEntryAttributeString( id_jobentry, "wildcard" );
onlyGettingNewFiles = rep.getJobEntryAttributeBoolean( id_jobentry, "only_new" );
usehttpproxy = rep.getJobEntryAttributeBoolean( id_jobentry, "usehttpproxy" );
httpProxyHost = rep.getJobEntryAttributeString( id_jobentry, "httpproxyhost" );
httpproxyport = rep.getJobEntryAttributeString( id_jobentry, "httpproxyport" );
httpproxyusername = rep.getJobEntryAttributeString( id_jobentry, "httpproxyusername" );
httpProxyPassword = rep.getJobEntryAttributeString( id_jobentry, "httpproxypassword" );
publicpublickey = rep.getJobEntryAttributeBoolean( id_jobentry, "publicpublickey" );
keyFilename = rep.getJobEntryAttributeString( id_jobentry, "keyfilename" );
keyFilePass = rep.getJobEntryAttributeString( id_jobentry, "keyfilepass" );
useBasicAuthentication = rep.getJobEntryAttributeBoolean( id_jobentry, "usebasicauthentication" );
afterFtpPut = rep.getJobEntryAttributeString( id_jobentry, "afterftpput" );
destinationfolder = rep.getJobEntryAttributeString( id_jobentry, "destinationfolder" );
createdestinationfolder = rep.getJobEntryAttributeBoolean( id_jobentry, "createdestinationfolder" );
cachehostkey = rep.getJobEntryAttributeBoolean( id_jobentry, "cachehostkey" );
timeout = (int) rep.getJobEntryAttributeInteger( id_jobentry, "timeout" );
createtargetfolder = rep.getJobEntryAttributeBoolean( id_jobentry, "createtargetfolder" );
includeSubFolders = rep.getJobEntryAttributeBoolean( id_jobentry, "includeSubFolders" );
} catch ( KettleException dbe ) {
throw new KettleException( BaseMessages.getString(
PKG, "JobSSH2GET.Log.UnableLoadRep", "" + id_jobentry, dbe.getMessage() ) );
}
}
@Override
public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_job ) throws KettleException {
try {
rep.saveJobEntryAttribute( id_job, getObjectId(), "servername", serverName );
rep.saveJobEntryAttribute( id_job, getObjectId(), "username", userName );
rep.saveJobEntryAttribute( id_job, getObjectId(), "password", Encr
.encryptPasswordIfNotUsingVariables( password ) );
rep.saveJobEntryAttribute( id_job, getObjectId(), "serverport", serverPort );
rep.saveJobEntryAttribute( id_job, getObjectId(), "ftpdirectory", ftpDirectory );
rep.saveJobEntryAttribute( id_job, getObjectId(), "localdirectory", localDirectory );
rep.saveJobEntryAttribute( id_job, getObjectId(), "wildcard", wildcard );
rep.saveJobEntryAttribute( id_job, getObjectId(), "only_new", onlyGettingNewFiles );
rep.saveJobEntryAttribute( id_job, getObjectId(), "usehttpproxy", usehttpproxy );
rep.saveJobEntryAttribute( id_job, getObjectId(), "httpproxyhost", httpProxyHost );
rep.saveJobEntryAttribute( id_job, getObjectId(), "httpproxyport", httpproxyport );
rep.saveJobEntryAttribute( id_job, getObjectId(), "httpproxyusername", httpproxyusername );
rep.saveJobEntryAttribute( id_job, getObjectId(), "httpproxypassword", httpProxyPassword );
rep.saveJobEntryAttribute( id_job, getObjectId(), "publicpublickey", publicpublickey );
rep.saveJobEntryAttribute( id_job, getObjectId(), "keyfilename", keyFilename );
rep.saveJobEntryAttribute( id_job, getObjectId(), "keyfilepass", keyFilePass );
rep.saveJobEntryAttribute( id_job, getObjectId(), "usebasicauthentication", useBasicAuthentication );
rep.saveJobEntryAttribute( id_job, getObjectId(), "afterftpput", afterFtpPut );
rep.saveJobEntryAttribute( id_job, getObjectId(), "destinationfolder", destinationfolder );
rep.saveJobEntryAttribute( id_job, getObjectId(), "createdestinationfolder", createdestinationfolder );
rep.saveJobEntryAttribute( id_job, getObjectId(), "cachehostkey", cachehostkey );
rep.saveJobEntryAttribute( id_job, getObjectId(), "timeout", timeout );
rep.saveJobEntryAttribute( id_job, getObjectId(), "createtargetfolder", createtargetfolder );
rep.saveJobEntryAttribute( id_job, getObjectId(), "includeSubFolders", includeSubFolders );
} catch ( KettleDatabaseException dbe ) {
throw new KettleException( BaseMessages.getString( PKG, "JobSSH2GET.Log.UnableSaveRep", "" + id_job, dbe
.getMessage() ) );
}
}
/**
* @return Returns the directory.
*/
public String getFtpDirectory() {
return ftpDirectory;
}
/**
* @param directory
* The directory to set.
*/
public void setFtpDirectory( String directory ) {
this.ftpDirectory = directory;
}
/**
* @return Returns the password.
*/
public String getPassword() {
return password;
}
/**
* @param password
* The password to set.
*/
public void setPassword( String password ) {
this.password = password;
}
/**
* @return Returns the afterftpput.
*/
public String getAfterFTPPut() {
return afterFtpPut;
}
/**
* @param afterFtpPut
* The action after (FTP/SSH) transfer to execute
*/
public void setAfterFTPPut( String afterFtpPut ) {
this.afterFtpPut = afterFtpPut;
}
/**
* @param proxyPassword
* The httpproxypassword to set.
*/
public void setHTTPProxyPassword( String proxyPassword ) {
this.httpProxyPassword = proxyPassword;
}
/**
* @return Returns the password.
*/
public String getHTTPProxyPassword() {
return httpProxyPassword;
}
/**
* @param keyFilePass
* The key file pass to set.
*/
public void setKeyFilePass( String keyFilePass ) {
this.keyFilePass = keyFilePass;
}
/**
* @return Returns the key file pass.
*/
public String getKeyFilePass() {
return keyFilePass;
}
/**
* @return Returns the serverName.
*/
public String getServerName() {
return serverName;
}
/**
* @param serverName
* The serverName to set.
*/
public void setServerName( String serverName ) {
this.serverName = serverName;
}
/**
* @param proxyhost
* The httpproxyhost to set.
*/
public void setHTTPProxyHost( String proxyhost ) {
this.httpProxyHost = proxyhost;
}
/**
* @return Returns the HTTP proxy host.
*/
public String getHTTPProxyHost() {
return httpProxyHost;
}
/**
* @param keyfilename
* The key filename to set.
*/
public void setKeyFilename( String keyfilename ) {
this.keyFilename = keyfilename;
}
/**
* @return Returns the key filename.
*/
public String getKeyFilename() {
return keyFilename;
}
/**
* @return Returns the userName.
*/
public String getUserName() {
return userName;
}
/**
* @param userName
* The userName to set.
*/
public void setUserName( String userName ) {
this.userName = userName;
}
/**
* @param proxyusername
* The httpproxyusername to set.
*/
public void setHTTPProxyUsername( String proxyusername ) {
this.httpproxyusername = proxyusername;
}
/**
* @return Returns the userName.
*/
public String getHTTPProxyUsername() {
return httpproxyusername;
}
/**
* @return Returns the wildcard.
*/
public String getWildcard() {
return wildcard;
}
/**
* @param wildcard
* The wildcard to set.
*/
public void setWildcard( String wildcard ) {
this.wildcard = wildcard;
}
/**
* @return Returns the localDirectory.
*/
public String getlocalDirectory() {
return localDirectory;
}
/**
* @param localDirectory
* The localDirectory to set.
*/
public void setlocalDirectory( String localDirectory ) {
this.localDirectory = localDirectory;
}
/**
* @return Returns the onlyGettingNewFiles.
*/
public boolean isOnlyGettingNewFiles() {
return onlyGettingNewFiles;
}
/**
* @param onlyGettingNewFiles
* The onlyGettingNewFiles to set.
*/
public void setOnlyGettingNewFiles( boolean onlyGettingNewFiles ) {
this.onlyGettingNewFiles = onlyGettingNewFiles;
}
/**
* @param cachehostkeyin
* The cachehostkey to set.
*/
public void setCacheHostKey( boolean cachehostkeyin ) {
this.cachehostkey = cachehostkeyin;
}
/**
* @return Returns the cachehostkey.
*/
public boolean isCacheHostKey() {
return cachehostkey;
}
/**
* @param httpproxy
* The usehttpproxy to set.
*/
public void setUseHTTPProxy( boolean httpproxy ) {
this.usehttpproxy = httpproxy;
}
/**
* @return Returns the usehttpproxy.
*/
public boolean isUseHTTPProxy() {
return usehttpproxy;
}
/**
* @return Returns the use basic authentication flag.
*/
public boolean isUseBasicAuthentication() {
return useBasicAuthentication;
}
/**
* @param useBasicAuthentication
* The use basic authentication flag to set.
*/
public void setUseBasicAuthentication( boolean useBasicAuthentication ) {
this.useBasicAuthentication = useBasicAuthentication;
}
/**
* @param includeSubFolders
* The include sub folders flag to set.
*/
public void setIncludeSubFolders( boolean includeSubFolders ) {
this.includeSubFolders = includeSubFolders;
}
/**
* @return Returns the include sub folders flag.
*/
public boolean isIncludeSubFolders() {
return includeSubFolders;
}
/**
* @param createdestinationfolderin
* The createdestinationfolder to set.
*/
public void setCreateDestinationFolder( boolean createdestinationfolderin ) {
this.createdestinationfolder = createdestinationfolderin;
}
/**
* @return Returns the createdestinationfolder.
*/
public boolean isCreateDestinationFolder() {
return createdestinationfolder;
}
/**
* @return Returns the CreateTargetFolder.
*/
public boolean isCreateTargetFolder() {
return createtargetfolder;
}
/**
* @param createtargetfolderin
* The createtargetfolder to set.
*/
public void setCreateTargetFolder( boolean createtargetfolderin ) {
this.createtargetfolder = createtargetfolderin;
}
/**
* @param publickey
* The publicpublickey to set.
*/
public void setUsePublicKey( boolean publickey ) {
this.publicpublickey = publickey;
}
/**
* @return Returns the usehttpproxy.
*/
public boolean isUsePublicKey() {
return publicpublickey;
}
public String getServerPort() {
return serverPort;
}
public void setServerPort( String serverPort ) {
this.serverPort = serverPort;
}
public void setHTTPProxyPort( String proxyport ) {
this.httpproxyport = proxyport;
}
public String getHTTPProxyPort() {
return httpproxyport;
}
public void setDestinationFolder( String destinationfolderin ) {
this.destinationfolder = destinationfolderin;
}
public String getDestinationFolder() {
return destinationfolder;
}
/**
* @param timeout
* The timeout to set.
*/
public void setTimeout( int timeout ) {
this.timeout = timeout;
}
/**
* @return Returns the timeout.
*/
public int getTimeout() {
return timeout;
}
@Override
public Result execute( Result previousResult, int nr ) {
Result result = previousResult;
result.setResult( false );
if ( log.isRowLevel() ) {
logRowlevel( BaseMessages.getString( PKG, "JobSSH2GET.Log.GettingFieldsValue" ) );
}
// Get real variable value
String realServerName = environmentSubstitute( serverName );
int realServerPort = Const.toInt( environmentSubstitute( serverPort ), 22 );
String realUserName = environmentSubstitute( userName );
String realServerPassword = Encr.decryptPasswordOptionallyEncrypted( environmentSubstitute( password ) );
// Proxy Host
String realProxyHost = environmentSubstitute( httpProxyHost );
int realProxyPort = Const.toInt( environmentSubstitute( httpproxyport ), 22 );
String realproxyUserName = environmentSubstitute( httpproxyusername );
String realProxyPassword =
Encr.decryptPasswordOptionallyEncrypted( environmentSubstitute( httpProxyPassword ) );
// Key file
String realKeyFilename = environmentSubstitute( keyFilename );
String relKeyFilepass = environmentSubstitute( keyFilePass );
// target files
String realLocalDirectory = environmentSubstitute( localDirectory );
String realwildcard = environmentSubstitute( wildcard );
// Remote source
String realftpDirectory = environmentSubstitute( ftpDirectory );
// Destination folder (Move to)
String realDestinationFolder = environmentSubstitute( destinationfolder );
try {
// Remote source
realftpDirectory = FTPUtils.normalizePath( realftpDirectory );
// Destination folder (Move to)
realDestinationFolder = FTPUtils.normalizePath( realDestinationFolder );
} catch ( Exception e ) {
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.CanNotNormalizePath", e.getMessage() ) );
result.setNrErrors( 1 );
return result;
}
// Check for mandatory fields
if ( log.isRowLevel() ) {
logRowlevel( BaseMessages.getString( PKG, "JobSSH2GET.Log.CheckingMandatoryFields" ) );
}
boolean mandatoryok = true;
if ( Utils.isEmpty( realServerName ) ) {
mandatoryok = false;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.ServernameMissing" ) );
}
if ( usehttpproxy ) {
if ( Utils.isEmpty( realProxyHost ) ) {
mandatoryok = false;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.HttpProxyhostMissing" ) );
}
}
if ( publicpublickey ) {
if ( Utils.isEmpty( realKeyFilename ) ) {
mandatoryok = false;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.KeyFileMissing" ) );
} else {
// Let's check if key file exists...
if ( !new File( realKeyFilename ).exists() ) {
mandatoryok = false;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.KeyFileNotExist" ) );
}
}
}
if ( Utils.isEmpty( realLocalDirectory ) ) {
mandatoryok = false;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.LocalFolderMissing" ) );
} else {
// Check if target folder exists...
if ( !new File( realLocalDirectory ).exists() ) {
if ( createtargetfolder ) {
// Create Target folder
if ( !CreateFolder( realLocalDirectory ) ) {
mandatoryok = false;
}
} else {
mandatoryok = false;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.LocalFolderNotExists", realLocalDirectory ) );
}
} else {
if ( !new File( realLocalDirectory ).isDirectory() ) {
mandatoryok = false;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.LocalFolderNotFolder", realLocalDirectory ) );
}
}
}
if ( afterFtpPut.equals( "move_file" ) ) {
if ( Utils.isEmpty( realDestinationFolder ) ) {
mandatoryok = false;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.DestinatFolderMissing" ) );
}
}
if ( mandatoryok ) {
Connection conn = null;
SFTPv3Client client = null;
boolean good = true;
try {
// Create a connection instance
conn =
getConnection(
realServerName, realServerPort, realProxyHost, realProxyPort, realproxyUserName, realProxyPassword );
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.ConnectionInstanceCreated" ) );
}
if ( timeout > 0 ) {
// Use timeout
// Cache Host Key
if ( cachehostkey ) {
conn.connect( new SimpleVerifier( database ), 0, timeout * 1000 );
} else {
conn.connect( null, 0, timeout * 1000 );
}
} else {
// Cache Host Key
if ( cachehostkey ) {
conn.connect( new SimpleVerifier( database ) );
} else {
conn.connect();
}
}
// Authenticate
boolean isAuthenticated = false;
if ( publicpublickey ) {
isAuthenticated =
conn.authenticateWithPublicKey( realUserName, new File( realKeyFilename ), relKeyFilepass );
} else {
isAuthenticated = conn.authenticateWithPassword( realUserName, realServerPassword );
}
// LET'S CHECK AUTHENTICATION ...
if ( isAuthenticated == false ) {
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.AuthenticationFailed" ) );
} else {
if ( log.isBasic() ) {
logBasic( BaseMessages.getString( PKG, "JobSSH2GET.Log.Connected", serverName, userName ) );
}
client = new SFTPv3Client( conn );
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.ProtocolVersion", ""
+ client.getProtocolVersion() ) );
}
// Check if ftp (source) directory exists
if ( !Utils.isEmpty( realftpDirectory ) ) {
if ( !sshDirectoryExists( client, realftpDirectory ) ) {
good = false;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.RemoteDirectoryNotExist", realftpDirectory ) );
} else if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.RemoteDirectoryExist", realftpDirectory ) );
}
}
if ( !Utils.isEmpty( realDestinationFolder ) ) {
// Check now destination folder
if ( !sshDirectoryExists( client, realDestinationFolder ) ) {
if ( createdestinationfolder ) {
if ( !CreateRemoteFolder( client, realDestinationFolder ) ) {
good = false;
}
} else {
good = false;
logError( BaseMessages.getString(
PKG, "JobSSH2GET.Log.DestinatFolderNotExist", realDestinationFolder ) );
}
}
}
if ( good ) {
Pattern pattern = null;
if ( !Utils.isEmpty( realwildcard ) ) {
pattern = Pattern.compile( realwildcard );
}
if ( includeSubFolders ) {
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.RecursiveModeOn" ) );
}
copyRecursive( realftpDirectory, realLocalDirectory, client, pattern, parentJob );
} else {
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.RecursiveModeOff" ) );
}
GetFiles( realftpDirectory, realLocalDirectory, client, pattern, parentJob );
}
/******************************** RESULT ********************/
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.Result.JobEntryEnd1" ) );
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.Result.TotalFiles", "" + nbfilestoget ) );
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.Result.TotalFilesPut", "" + nbgot ) );
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.Result.TotalFilesError", "" + nbrerror ) );
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.Result.JobEntryEnd2" ) );
}
if ( nbrerror == 0 ) {
result.setResult( true );
/******************************** RESULT ********************/
}
}
}
} catch ( Exception e ) {
result.setNrErrors( nbrerror );
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.ErrorFTP", e.getMessage() ) );
} finally {
if ( conn != null ) {
conn.close();
}
if ( client != null ) {
client.close();
}
}
}
return result;
}
private Connection getConnection( String servername, int serverport, String proxyhost, int proxyport,
String proxyusername, String proxypassword ) {
/* Create a connection instance */
Connection conn = new Connection( servername, serverport );
/* We want to connect through a HTTP proxy */
if ( usehttpproxy ) {
conn.setProxyData( new HTTPProxyData( proxyhost, proxyport ) );
/* Now connect */
// if the proxy requires basic authentication:
if ( useBasicAuthentication ) {
conn.setProxyData( new HTTPProxyData( proxyhost, proxyport, proxyusername, proxypassword ) );
}
}
return conn;
}
/**
* Check existence of a file
*
* @param sftpClient
* @param filename
* @return true, if file exists
* @throws Exception
*/
public boolean sshFileExists( SFTPv3Client sftpClient, String filename ) {
try {
SFTPv3FileAttributes attributes = sftpClient.stat( filename );
if ( attributes != null ) {
return ( attributes.isRegularFile() );
} else {
return false;
}
} catch ( Exception e ) {
return false;
}
}
/**
* Check existence of a local file
*
* @param filename
* @return true, if file exists
*/
public boolean FileExists( String filename ) {
FileObject file = null;
try {
file = KettleVFS.getFileObject( filename, this );
if ( !file.exists() ) {
return false;
} else {
if ( file.getType() == FileType.FILE ) {
return true;
} else {
return false;
}
}
} catch ( Exception e ) {
return false;
}
}
/**
* Checks if file is a directory
*
* @param sftpClient
* @param filename
* @return true, if filename is a directory
*/
public boolean isDirectory( SFTPv3Client sftpClient, String filename ) {
try {
return sftpClient.stat( filename ).isDirectory();
} catch ( Exception e ) {
// Ignore errors
}
return false;
}
/**
* Checks if a directory exists
*
* @param sftpClient
* @param directory
* @return true, if directory exists
*/
public boolean sshDirectoryExists( SFTPv3Client sftpClient, String directory ) {
try {
SFTPv3FileAttributes attributes = sftpClient.stat( directory );
if ( attributes != null ) {
return ( attributes.isDirectory() );
} else {
return false;
}
} catch ( Exception e ) {
return false;
}
}
/**
* Returns the file size of a file
*
* @param sftpClient
* @param filename
* @return the size of the file
* @throws Exception
*/
public long getFileSize( SFTPv3Client sftpClient, String filename ) throws Exception {
return sftpClient.stat( filename ).size.longValue();
}
/**********************************************************
*
* @param selectedfile
* @param wildcard
* @param pattern
* @return True if the selectedfile matches the wildcard
**********************************************************/
private boolean getFileWildcard( String selectedfile, Pattern pattern ) {
boolean getIt = true;
// First see if the file matches the regular expression!
if ( pattern != null ) {
Matcher matcher = pattern.matcher( selectedfile );
getIt = matcher.matches();
}
return getIt;
}
private boolean deleteOrMoveFiles( SFTPv3Client sftpClient, String filename, String destinationFolder ) {
boolean retval = false;
// Delete the file if this is needed!
if ( afterFtpPut.equals( "delete_file" ) ) {
try {
sftpClient.rm( filename );
retval = true;
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.DeletedFile", filename ) );
}
} catch ( Exception e ) {
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.CanNotDeleteRemoteFile", filename ), e );
}
} else if ( afterFtpPut.equals( "move_file" ) ) {
String DestinationFullFilename = destinationFolder + Const.FILE_SEPARATOR + filename;
try {
sftpClient.mv( filename, DestinationFullFilename );
retval = true;
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.DeletedFile", filename ) );
}
} catch ( Exception e ) {
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.MovedFile", filename, destinationFolder ), e );
}
}
return retval;
}
/**
* copy a directory from the remote host to the local one.
*
* @param sourceLocation
* the source directory on the remote host
* @param targetLocation
* the target directory on the local host
* @param sftpClient
* is an instance of SFTPv3Client that makes SFTP client connection over SSH-2
* @return the number of files successfully copied
* @throws Exception
*/
@SuppressWarnings( "unchecked" )
private void GetFiles( String sourceLocation, String targetLocation, SFTPv3Client sftpClient, Pattern pattern,
Job parentJob ) throws Exception {
String sourceFolder = ".";
if ( !Utils.isEmpty( sourceLocation ) ) {
sourceFolder = sourceLocation + FTPUtils.FILE_SEPARATOR;
} else {
sourceFolder += FTPUtils.FILE_SEPARATOR;
}
Vector<SFTPv3DirectoryEntry> filelist = sftpClient.ls( sourceFolder );
if ( filelist != null ) {
Iterator<SFTPv3DirectoryEntry> iterator = filelist.iterator();
while ( iterator.hasNext() && !parentJob.isStopped() ) {
SFTPv3DirectoryEntry dirEntry = iterator.next();
if ( dirEntry == null ) {
continue;
}
if ( dirEntry.filename.equals( "." )
|| dirEntry.filename.equals( ".." ) || isDirectory( sftpClient, sourceFolder + dirEntry.filename ) ) {
continue;
}
if ( getFileWildcard( dirEntry.filename, pattern ) ) {
// Copy file from remote host
copyFile(
sourceFolder + dirEntry.filename, targetLocation + FTPUtils.FILE_SEPARATOR + dirEntry.filename,
sftpClient );
}
}
}
}
/**
* copy a directory from the remote host to the local one recursivly.
*
* @param sourceLocation
* the source directory on the remote host
* @param targetLocation
* the target directory on the local host
* @param sftpClient
* is an instance of SFTPv3Client that makes SFTP client connection over SSH-2
* @return the number of files successfully copied
* @throws Exception
*/
private void copyRecursive( String sourceLocation, String targetLocation, SFTPv3Client sftpClient,
Pattern pattern, Job parentJob ) throws Exception {
String sourceFolder = "." + FTPUtils.FILE_SEPARATOR;
if ( sourceLocation != null ) {
sourceFolder = sourceLocation;
}
if ( this.isDirectory( sftpClient, sourceFolder ) ) {
Vector<?> filelist = sftpClient.ls( sourceFolder );
Iterator<?> iterator = filelist.iterator();
while ( iterator.hasNext() ) {
SFTPv3DirectoryEntry dirEntry = (SFTPv3DirectoryEntry) iterator.next();
if ( dirEntry == null ) {
continue;
}
if ( dirEntry.filename.equals( "." ) || dirEntry.filename.equals( ".." ) ) {
continue;
}
copyRecursive( sourceFolder + FTPUtils.FILE_SEPARATOR + dirEntry.filename, targetLocation
+ Const.FILE_SEPARATOR + dirEntry.filename, sftpClient, pattern, parentJob );
}
} else if ( isFile( sftpClient, sourceFolder ) ) {
if ( getFileWildcard( sourceFolder, pattern ) ) {
copyFile( sourceFolder, targetLocation, sftpClient );
}
}
}
/**
* Checks if file is a file
*
* @param sftpClient
* @param filename
* @return true, if filename is a directory
*/
public boolean isFile( SFTPv3Client sftpClient, String filename ) {
try {
return sftpClient.stat( filename ).isRegularFile();
} catch ( Exception e ) {
// Ignore errors
}
return false;
}
/**
*
* @param sourceLocation
* @param targetLocation
* @param sftpClient
* @return
*/
private void copyFile( String sourceLocation, String targetLocation, SFTPv3Client sftpClient ) {
SFTPv3FileHandle sftpFileHandle = null;
FileOutputStream fos = null;
File transferFile = null;
long remoteFileSize = -1;
boolean filecopied = true;
try {
transferFile = new File( targetLocation );
if ( ( onlyGettingNewFiles == false )
|| ( onlyGettingNewFiles == true ) && !FileExists( transferFile.getAbsolutePath() ) ) {
new File( transferFile.getParent() ).mkdirs();
remoteFileSize = this.getFileSize( sftpClient, sourceLocation );
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.ReceivingFile", sourceLocation, transferFile
.getAbsolutePath(), "" + remoteFileSize ) );
}
sftpFileHandle = sftpClient.openFileRO( sourceLocation );
fos = null;
long offset = 0;
fos = new FileOutputStream( transferFile );
byte[] buffer = new byte[2048];
while ( true ) {
int len = sftpClient.read( sftpFileHandle, offset, buffer, 0, buffer.length );
if ( len <= 0 ) {
break;
}
fos.write( buffer, 0, len );
offset += len;
}
fos.flush();
fos.close();
fos = null;
nbfilestoget++;
if ( remoteFileSize > 0 && remoteFileSize != transferFile.length() ) {
filecopied = false;
nbrerror++;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.RemoteFileLocalDifferent", ""
+ remoteFileSize, transferFile.length() + "", "" + offset ) );
} else {
nbgot++;
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString(
PKG, "JobSSH2GET.Log.RemoteFileLocalCopied", sourceLocation, transferFile + "" ) );
}
}
}
// Let's now delete or move file if needed...
if ( filecopied && !afterFtpPut.equals( "do_nothing" ) ) {
deleteOrMoveFiles( sftpClient, sourceLocation, environmentSubstitute( destinationfolder ) );
}
} catch ( Exception e ) {
nbrerror++;
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.WritingFile", transferFile.getAbsolutePath(), e
.getMessage() ) );
} finally {
try {
if ( sftpFileHandle != null ) {
sftpClient.closeFile( sftpFileHandle );
sftpFileHandle = null;
}
if ( fos != null ) {
try {
fos.close();
fos = null;
} catch ( Exception ex ) {
// Ignore errors
}
}
} catch ( Exception e ) {
// Ignore errors
}
}
}
private boolean CreateFolder( String filefolder ) {
FileObject folder = null;
try {
folder = KettleVFS.getFileObject( filefolder, this );
if ( !folder.exists() ) {
if ( createtargetfolder ) {
folder.createFolder();
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.FolderCreated", folder.toString() ) );
}
} else {
return false;
}
}
return true;
} catch ( Exception e ) {
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.CanNotCreateFolder", folder.toString() ), e );
} finally {
if ( folder != null ) {
try {
folder.close();
} catch ( Exception ex ) { /* Ignore */
}
}
}
return false;
}
/**
* Create remote folder
*
* @param sftpClient
* @param foldername
* @return true, if foldername is created
*/
private boolean CreateRemoteFolder( SFTPv3Client sftpClient, String foldername ) {
boolean retval = false;
if ( !sshDirectoryExists( sftpClient, foldername ) ) {
try {
sftpClient.mkdir( foldername, 0700 );
retval = true;
if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.RemoteFolderCreated", foldername ) );
}
} catch ( Exception e ) {
logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.CreatingRemoteFolder", foldername ), e );
}
}
return retval;
}
@Override
public boolean evaluates() {
return true;
}
@Override
public List<ResourceReference> getResourceDependencies( JobMeta jobMeta ) {
List<ResourceReference> references = super.getResourceDependencies( jobMeta );
if ( !Utils.isEmpty( serverName ) ) {
String realServerName = jobMeta.environmentSubstitute( serverName );
ResourceReference reference = new ResourceReference( this );
reference.getEntries().add( new ResourceEntry( realServerName, ResourceType.SERVER ) );
references.add( reference );
}
return references;
}
@Override
public void check( List<CheckResultInterface> remarks, JobMeta jobMeta, VariableSpace space,
Repository repository, IMetaStore metaStore ) {
JobEntryValidatorUtils.andValidator().validate( this, "serverName", remarks,
AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator() ) );
JobEntryValidatorUtils.andValidator().validate( this, "localDirectory", remarks,
AndValidator.putValidators(
JobEntryValidatorUtils.notBlankValidator(),
JobEntryValidatorUtils.fileExistsValidator() ) );
JobEntryValidatorUtils.andValidator().validate( this, "userName", remarks,
AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator() ) );
JobEntryValidatorUtils.andValidator().validate( this, "password", remarks,
AndValidator.putValidators( JobEntryValidatorUtils.notNullValidator() ) );
JobEntryValidatorUtils.andValidator().validate( this, "serverPort", remarks,
AndValidator.putValidators( JobEntryValidatorUtils.integerValidator() ) );
}
}