/*! ****************************************************************************** * * Pentaho Data Integration * * Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com * ******************************************************************************* * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ******************************************************************************/ package org.pentaho.di.job.entries.ftpsput; import org.pentaho.di.job.entry.validator.AndValidator; import org.pentaho.di.job.entry.validator.JobEntryValidatorUtils; import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.pentaho.di.cluster.SlaveServer; import org.pentaho.di.core.CheckResultInterface; import org.pentaho.di.core.Const; import org.pentaho.di.core.util.Utils; import org.pentaho.di.core.Result; import org.pentaho.di.core.database.DatabaseMeta; import org.pentaho.di.core.encryption.Encr; import org.pentaho.di.core.exception.KettleDatabaseException; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.exception.KettleXMLException; import org.pentaho.di.core.variables.VariableSpace; import org.pentaho.di.core.xml.XMLHandler; import org.pentaho.di.i18n.BaseMessages; import org.pentaho.di.job.JobMeta; import org.pentaho.di.job.entries.ftpsget.FTPSConnection; import org.pentaho.di.job.entry.JobEntryBase; import org.pentaho.di.job.entry.JobEntryInterface; import org.pentaho.di.repository.ObjectId; import org.pentaho.di.repository.Repository; import org.pentaho.di.resource.ResourceEntry; import org.pentaho.di.resource.ResourceEntry.ResourceType; import org.pentaho.di.resource.ResourceReference; import org.pentaho.metastore.api.IMetaStore; import org.w3c.dom.Node; /** * This defines an FTPS put job entry. * * @author Samatar * @since 15-03-2010 * */ public class JobEntryFTPSPUT extends JobEntryBase implements Cloneable, JobEntryInterface { private static Class<?> PKG = JobEntryFTPSPUT.class; // for i18n purposes, needed by Translator2!! private String serverName; private String serverPort; private String userName; private String password; private String remoteDirectory; private String localDirectory; private String wildcard; private boolean binaryMode; private int timeout; private boolean remove; private boolean onlyPuttingNewFiles; /* Don't overwrite files */ private boolean activeConnection; private String proxyHost; private String proxyPort; /* string to allow variable substitution */ private String proxyUsername; private String proxyPassword; private int connectionType; public JobEntryFTPSPUT( String n ) { super( n, "" ); serverName = null; serverPort = "21"; remoteDirectory = null; localDirectory = null; connectionType = FTPSConnection.CONNECTION_TYPE_FTP; } public JobEntryFTPSPUT() { this( "" ); } public Object clone() { JobEntryFTPSPUT je = (JobEntryFTPSPUT) super.clone(); return je; } public String getXML() { StringBuilder retval = new StringBuilder( 400 ); retval.append( super.getXML() ); retval.append( " " ).append( XMLHandler.addTagValue( "servername", serverName ) ); retval.append( " " ).append( XMLHandler.addTagValue( "serverport", serverPort ) ); retval.append( " " ).append( XMLHandler.addTagValue( "username", userName ) ); retval.append( " " ).append( XMLHandler.addTagValue( "password", Encr.encryptPasswordIfNotUsingVariables( getPassword() ) ) ); retval.append( " " ).append( XMLHandler.addTagValue( "remoteDirectory", remoteDirectory ) ); retval.append( " " ).append( XMLHandler.addTagValue( "localDirectory", localDirectory ) ); retval.append( " " ).append( XMLHandler.addTagValue( "wildcard", wildcard ) ); retval.append( " " ).append( XMLHandler.addTagValue( "binary", binaryMode ) ); retval.append( " " ).append( XMLHandler.addTagValue( "timeout", timeout ) ); retval.append( " " ).append( XMLHandler.addTagValue( "remove", remove ) ); retval.append( " " ).append( XMLHandler.addTagValue( "only_new", onlyPuttingNewFiles ) ); retval.append( " " ).append( XMLHandler.addTagValue( "active", activeConnection ) ); retval.append( " " ).append( XMLHandler.addTagValue( "proxy_host", proxyHost ) ); retval.append( " " ).append( XMLHandler.addTagValue( "proxy_port", proxyPort ) ); retval.append( " " ).append( XMLHandler.addTagValue( "proxy_username", proxyUsername ) ); retval.append( " " ).append( XMLHandler.addTagValue( "proxy_password", proxyPassword ) ); retval.append( " " ).append( XMLHandler.addTagValue( "connection_type", FTPSConnection.getConnectionTypeCode( connectionType ) ) ); return retval.toString(); } public void loadXML( Node entrynode, List<DatabaseMeta> databases, List<SlaveServer> slaveServers, Repository rep, IMetaStore metaStore ) throws KettleXMLException { try { super.loadXML( entrynode, databases, slaveServers ); serverName = XMLHandler.getTagValue( entrynode, "servername" ); serverPort = XMLHandler.getTagValue( entrynode, "serverport" ); userName = XMLHandler.getTagValue( entrynode, "username" ); password = Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( entrynode, "password" ) ); remoteDirectory = XMLHandler.getTagValue( entrynode, "remoteDirectory" ); localDirectory = XMLHandler.getTagValue( entrynode, "localDirectory" ); wildcard = XMLHandler.getTagValue( entrynode, "wildcard" ); binaryMode = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "binary" ) ); timeout = Const.toInt( XMLHandler.getTagValue( entrynode, "timeout" ), 10000 ); remove = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "remove" ) ); onlyPuttingNewFiles = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "only_new" ) ); activeConnection = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "active" ) ); proxyHost = XMLHandler.getTagValue( entrynode, "proxy_host" ); proxyPort = XMLHandler.getTagValue( entrynode, "proxy_port" ); proxyUsername = XMLHandler.getTagValue( entrynode, "proxy_username" ); proxyPassword = XMLHandler.getTagValue( entrynode, "proxy_password" ); connectionType = FTPSConnection.getConnectionTypeByCode( Const.NVL( XMLHandler.getTagValue( entrynode, "connection_type" ), "" ) ); } catch ( KettleXMLException xe ) { throw new KettleXMLException( BaseMessages.getString( PKG, "JobFTPSPUT.Log.UnableToLoadFromXml" ), xe ); } } public void loadRep( Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List<DatabaseMeta> databases, List<SlaveServer> slaveServers ) throws KettleException { try { serverName = rep.getJobEntryAttributeString( id_jobentry, "servername" ); serverPort = rep.getJobEntryAttributeString( id_jobentry, "serverport" ); userName = rep.getJobEntryAttributeString( id_jobentry, "username" ); password = Encr.decryptPasswordOptionallyEncrypted( rep.getJobEntryAttributeString( id_jobentry, "password" ) ); remoteDirectory = rep.getJobEntryAttributeString( id_jobentry, "remoteDirectory" ); localDirectory = rep.getJobEntryAttributeString( id_jobentry, "localDirectory" ); wildcard = rep.getJobEntryAttributeString( id_jobentry, "wildcard" ); binaryMode = rep.getJobEntryAttributeBoolean( id_jobentry, "binary" ); timeout = (int) rep.getJobEntryAttributeInteger( id_jobentry, "timeout" ); remove = rep.getJobEntryAttributeBoolean( id_jobentry, "remove" ); onlyPuttingNewFiles = rep.getJobEntryAttributeBoolean( id_jobentry, "only_new" ); activeConnection = rep.getJobEntryAttributeBoolean( id_jobentry, "active" ); proxyHost = rep.getJobEntryAttributeString( id_jobentry, "proxy_host" ); proxyPort = rep.getJobEntryAttributeString( id_jobentry, "proxy_port" ); proxyUsername = rep.getJobEntryAttributeString( id_jobentry, "proxy_username" ); proxyPassword = rep.getJobEntryAttributeString( id_jobentry, "proxy_password" ); connectionType = FTPSConnection.getConnectionTypeByCode( Const.NVL( rep.getJobEntryAttributeString( id_jobentry, "connection_type" ), "" ) ); } catch ( KettleException dbe ) { throw new KettleException( BaseMessages.getString( PKG, "JobFTPSPUT.UnableToLoadFromRepo", String .valueOf( id_jobentry ) ), dbe ); } } public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_job ) throws KettleException { try { rep.saveJobEntryAttribute( id_job, getObjectId(), "servername", serverName ); rep.saveJobEntryAttribute( id_job, getObjectId(), "serverport", serverPort ); rep.saveJobEntryAttribute( id_job, getObjectId(), "username", userName ); rep.saveJobEntryAttribute( id_job, getObjectId(), "password", Encr .encryptPasswordIfNotUsingVariables( password ) ); rep.saveJobEntryAttribute( id_job, getObjectId(), "remoteDirectory", remoteDirectory ); rep.saveJobEntryAttribute( id_job, getObjectId(), "localDirectory", localDirectory ); rep.saveJobEntryAttribute( id_job, getObjectId(), "wildcard", wildcard ); rep.saveJobEntryAttribute( id_job, getObjectId(), "binary", binaryMode ); rep.saveJobEntryAttribute( id_job, getObjectId(), "timeout", timeout ); rep.saveJobEntryAttribute( id_job, getObjectId(), "remove", remove ); rep.saveJobEntryAttribute( id_job, getObjectId(), "only_new", onlyPuttingNewFiles ); rep.saveJobEntryAttribute( id_job, getObjectId(), "active", activeConnection ); rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_host", proxyHost ); rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_port", proxyPort ); rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_username", proxyUsername ); rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_password", proxyPassword ); rep.saveJobEntryAttribute( id_job, getObjectId(), "connection_type", FTPSConnection .getConnectionType( connectionType ) ); } catch ( KettleDatabaseException dbe ) { throw new KettleException( BaseMessages.getString( PKG, "JobFTPSPUT.UnableToSaveToRepo", String .valueOf( id_job ) ), dbe ); } } /** * @return Returns the binaryMode. */ public boolean isBinaryMode() { return binaryMode; } /** * @param binaryMode * The binaryMode to set. */ public void setBinaryMode( boolean binaryMode ) { this.binaryMode = binaryMode; } /** * @param timeout * The timeout to set. */ public void setTimeout( int timeout ) { this.timeout = timeout; } /** * @return Returns the timeout. */ public int getTimeout() { return timeout; } /** * @return Returns the onlyGettingNewFiles. */ public boolean isOnlyPuttingNewFiles() { return onlyPuttingNewFiles; } /** * @param onlyPuttingNewFiles * Only transfer new files to the remote host */ public void setOnlyPuttingNewFiles( boolean onlyPuttingNewFiles ) { this.onlyPuttingNewFiles = onlyPuttingNewFiles; } /** * @return Returns the remoteDirectory. */ public String getRemoteDirectory() { return remoteDirectory; } /** * @param directory * The remoteDirectory to set. */ public void setRemoteDirectory( String directory ) { this.remoteDirectory = directory; } /** * @return Returns the password. */ public String getPassword() { return password; } /** * @param password * The password to set. */ public void setPassword( String password ) { this.password = password; } /** * @return Returns the serverName. */ public String getServerName() { return serverName; } /** * @param serverName * The serverName to set. */ public void setServerName( String serverName ) { this.serverName = serverName; } /** * @return Returns the userName. */ public String getUserName() { return userName; } /** * @param userName * The userName to set. */ public void setUserName( String userName ) { this.userName = userName; } /** * @return Returns the wildcard. */ public String getWildcard() { return wildcard; } /** * @param wildcard * The wildcard to set. */ public void setWildcard( String wildcard ) { this.wildcard = wildcard; } /** * @return Returns the localDirectory. */ public String getLocalDirectory() { return localDirectory; } /** * @param directory * The localDirectory to set. */ public void setLocalDirectory( String directory ) { this.localDirectory = directory; } /** * @param remove * The remove to set. */ public void setRemove( boolean remove ) { this.remove = remove; } /** * @return Returns the remove. */ public boolean getRemove() { return remove; } public String getServerPort() { return serverPort; } public void setServerPort( String serverPort ) { this.serverPort = serverPort; } /** * @return the activeConnection */ public boolean isActiveConnection() { return activeConnection; } /** * @param activeConnection * set to true to get an active FTP connection */ public void setActiveConnection( boolean activeConnection ) { this.activeConnection = activeConnection; } /** * @return Returns the hostname of the ftp-proxy. */ public String getProxyHost() { return proxyHost; } /** * @param proxyHost * The hostname of the proxy. */ public void setProxyHost( String proxyHost ) { this.proxyHost = proxyHost; } /** * @return Returns the password which is used to authenticate at the proxy. */ public String getProxyPassword() { return proxyPassword; } /** * @param proxyPassword * The password which is used to authenticate at the proxy. */ public void setProxyPassword( String proxyPassword ) { this.proxyPassword = proxyPassword; } /** * @return Returns the port of the ftp-proxy. */ public String getProxyPort() { return proxyPort; } /** * @return the conenction type */ public int getConnectionType() { return connectionType; } /** * @param connectionType * the connectionType to set */ public void setConnectionType( int type ) { connectionType = type; } /** * @param proxyPort * The port of the ftp-proxy. */ public void setProxyPort( String proxyPort ) { this.proxyPort = proxyPort; } /** * @return Returns the username which is used to authenticate at the proxy. */ public String getProxyUsername() { return proxyUsername; } /** * @param proxyUsername * The username which is used to authenticate at the proxy. */ public void setProxyUsername( String proxyUsername ) { this.proxyUsername = proxyUsername; } public Result execute( Result previousResult, int nr ) { Result result = previousResult; result.setResult( false ); long filesput = 0; if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobFTPSPUT.Log.Starting" ) ); } // String substitution.. String realServerName = environmentSubstitute( serverName ); String realServerPort = environmentSubstitute( serverPort ); String realUsername = environmentSubstitute( userName ); String realPassword = Encr.decryptPasswordOptionallyEncrypted( environmentSubstitute( password ) ); String realRemoteDirectory = environmentSubstitute( remoteDirectory ); String realWildcard = environmentSubstitute( wildcard ); String realLocalDirectory = environmentSubstitute( localDirectory ); FTPSConnection connection = null; try { // Create FTPS client to host:port ... int realPort = Const.toInt( environmentSubstitute( realServerPort ), 0 ); // Define a new connection connection = new FTPSConnection( getConnectionType(), realServerName, realPort, realUsername, realPassword ); this.buildFTPSConnection( connection ); // move to spool dir ... if ( !Utils.isEmpty( realRemoteDirectory ) ) { connection.changeDirectory( realRemoteDirectory ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobFTPSPUT.Log.ChangedDirectory", realRemoteDirectory ) ); } } realRemoteDirectory = Const.NVL( realRemoteDirectory, FTPSConnection.HOME_FOLDER ); ArrayList<String> myFileList = new ArrayList<String>(); File localFiles = new File( realLocalDirectory ); if ( !localFiles.exists() ) { // if local directory uses ${ signature this will be fail to MessageFormat.format ... String error = BaseMessages.getString( PKG, "JobFTPSPUT.LocalFileDirectoryNotExists" ) + realLocalDirectory; throw new Exception( error ); } File[] children = localFiles.listFiles(); for ( int i = 0; i < children.length; i++ ) { // Get filename of file or directory if ( !children[i].isDirectory() ) { myFileList.add( children[i].getName() ); } } String[] filelist = new String[myFileList.size()]; myFileList.toArray( filelist ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobFTPSPUT.Log.FoundFileLocalDirectory", filelist.length, realLocalDirectory ) ); } Pattern pattern = null; if ( !Utils.isEmpty( realWildcard ) ) { pattern = Pattern.compile( realWildcard ); } // end if // Get the files in the list and execute put each file in the FTP for ( int i = 0; i < filelist.length && !parentJob.isStopped(); i++ ) { boolean getIt = true; // First see if the file matches the regular expression! if ( pattern != null ) { Matcher matcher = pattern.matcher( filelist[i] ); getIt = matcher.matches(); } if ( getIt ) { // File exists? boolean fileExist = connection.isFileExists( filelist[i] ); if ( isDebug() ) { if ( fileExist ) { logDebug( BaseMessages.getString( PKG, "JobFTPSPUT.Log.FileExists", filelist[i] ) ); } else { logDebug( BaseMessages.getString( PKG, "JobFTPSPUT.Log.FileDoesNotExists", filelist[i] ) ); } } if ( !fileExist || ( !onlyPuttingNewFiles && fileExist ) ) { String localFilename = realLocalDirectory + Const.FILE_SEPARATOR + filelist[i]; if ( isDebug() ) { logDebug( BaseMessages.getString( PKG, "JobFTPSPUT.Log.PuttingFileToRemoteDirectory", localFilename, realRemoteDirectory ) ); } connection.uploadFile( localFilename, filelist[i] ); filesput++; // Delete the file if this is needed! if ( remove ) { new File( localFilename ).delete(); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobFTPSPUT.Log.DeletedFile", localFilename ) ); } } } } } result.setResult( true ); result.setNrLinesOutput( filesput ); if ( isDetailed() ) { logDebug( BaseMessages.getString( PKG, "JobFTPSPUT.Log.WeHavePut", filesput ) ); } } catch ( Exception e ) { result.setNrErrors( 1 ); logError( BaseMessages.getString( PKG, "JobFTPSPUT.Log.ErrorPuttingFiles", e.getMessage() ) ); logError( Const.getStackTracker( e ) ); } finally { if ( connection != null ) { try { connection.disconnect(); } catch ( Exception e ) { logError( BaseMessages.getString( PKG, "JobFTPSPUT.Log.ErrorQuitingFTP", e.getMessage() ) ); } } } return result; } public boolean evaluates() { return true; } public List<ResourceReference> getResourceDependencies( JobMeta jobMeta ) { List<ResourceReference> references = super.getResourceDependencies( jobMeta ); if ( !Utils.isEmpty( serverName ) ) { String realServerName = jobMeta.environmentSubstitute( serverName ); ResourceReference reference = new ResourceReference( this ); reference.getEntries().add( new ResourceEntry( realServerName, ResourceType.SERVER ) ); references.add( reference ); } return references; } @Override public void check( List<CheckResultInterface> remarks, JobMeta jobMeta, VariableSpace space, Repository repository, IMetaStore metaStore ) { JobEntryValidatorUtils.andValidator().validate( this, "serverName", remarks, AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator() ) ); JobEntryValidatorUtils.andValidator().validate( this, "localDirectory", remarks, AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator(), JobEntryValidatorUtils.fileExistsValidator() ) ); JobEntryValidatorUtils.andValidator().validate( this, "userName", remarks, AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator() ) ); JobEntryValidatorUtils.andValidator().validate( this, "password", remarks, AndValidator.putValidators( JobEntryValidatorUtils.notNullValidator() ) ); JobEntryValidatorUtils.andValidator().validate( this, "serverPort", remarks, AndValidator.putValidators( JobEntryValidatorUtils.integerValidator() ) ); } void buildFTPSConnection( FTPSConnection connection ) throws Exception { if ( !Utils.isEmpty( proxyHost ) ) { String realProxy_host = environmentSubstitute( proxyHost ); String realProxy_username = environmentSubstitute( proxyUsername ); String realProxy_password = environmentSubstitute( proxyPassword ); realProxy_password = Encr.decryptPasswordOptionallyEncrypted( realProxy_password ); connection.setProxyHost( realProxy_host ); if ( !Utils.isEmpty( realProxy_username ) ) { connection.setProxyUser( realProxy_username ); } if ( !Utils.isEmpty( realProxy_password ) ) { connection.setProxyPassword( realProxy_password ); } if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobEntryFTPSPUT.OpenedProxyConnectionOn", realProxy_host ) ); } int proxyport = Const.toInt( environmentSubstitute( proxyPort ), 21 ); if ( proxyport != 0 ) { connection.setProxyPort( proxyport ); } } else { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobEntryFTPSPUT.OpenedConnectionTo", connection.getHostName() ) ); } } // set activeConnection connectmode ... if ( activeConnection ) { connection.setPassiveMode( false ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobFTPSPUT.Log.SetActiveConnection" ) ); } } else { connection.setPassiveMode( true ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobFTPSPUT.Log.SetPassiveConnection" ) ); } } // Set the timeout connection.setTimeOut( timeout ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobFTPSPUT.Log.SetTimeout", timeout ) ); } // login to FTPS host ... connection.connect(); if ( isDetailed() ) { // Remove password from logging, you don't know where it ends up. logDetailed( BaseMessages.getString( PKG, "JobFTPSPUT.Log.Logged", connection.getUserName() ) ); logDetailed( BaseMessages.getString( PKG, "JobFTPSPUT.WorkingDirectory", connection.getWorkingDirectory() ) ); } // Set binary mode if ( isBinaryMode() ) { connection.setBinaryMode( true ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobFTPSPUT.Log.BinaryMod" ) ); } } } }