/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nifi.processors.standard.util; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Properties; import java.util.Vector; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.PropertyValue; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.util.StandardValidators; import org.slf4j.LoggerFactory; import com.jcraft.jsch.ChannelSftp; import com.jcraft.jsch.ChannelSftp.LsEntry; import com.jcraft.jsch.ChannelSftp.LsEntrySelector; import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; import com.jcraft.jsch.SftpException; public class SFTPTransfer implements FileTransfer { public static final PropertyDescriptor PRIVATE_KEY_PATH = new PropertyDescriptor.Builder() .name("Private Key Path") .description("The fully qualified path to the Private Key file") .required(false) .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) .build(); public static final PropertyDescriptor PRIVATE_KEY_PASSPHRASE = new PropertyDescriptor.Builder() .name("Private Key Passphrase") .description("Password for the private key") .required(false) .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .sensitive(true) .build(); public static final PropertyDescriptor HOST_KEY_FILE = new PropertyDescriptor.Builder() .name("Host Key File") .description("If supplied, the given file will be used as the Host Key; otherwise, no use host key file will be used") .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) .required(false) .build(); public static final PropertyDescriptor STRICT_HOST_KEY_CHECKING = new PropertyDescriptor.Builder() .name("Strict Host Key Checking") .description("Indicates whether or not strict enforcement of hosts keys should be applied") .allowableValues("true", "false") .defaultValue("false") .required(true) .build(); public static final PropertyDescriptor PORT = new PropertyDescriptor.Builder() .name("Port") .description("The port that the remote system is listening on for file transfers") .addValidator(StandardValidators.PORT_VALIDATOR) .required(true) .defaultValue("22") .build(); public static final PropertyDescriptor USE_KEEPALIVE_ON_TIMEOUT = new PropertyDescriptor.Builder() .name("Send Keep Alive On Timeout") .description("Indicates whether or not to send a single Keep Alive message when SSH socket times out") .allowableValues("true", "false") .defaultValue("true") .required(true) .build(); /** * Dynamic property which is used to decide if the {@link #ensureDirectoryExists(FlowFile, File)} method should perform a {@link ChannelSftp#ls(String)} before calling * {@link ChannelSftp#mkdir(String)}. In most cases, the code should call ls before mkdir, but some weird permission setups (chmod 100) on a directory would cause the 'ls' to throw a permission * exception. * <p> * This property is dynamic until deemed a worthy inclusion as proper. */ public static final PropertyDescriptor DISABLE_DIRECTORY_LISTING = new PropertyDescriptor.Builder() .name("Disable Directory Listing") .description("Disables directory listings before operations which might fail, such as configurations which create directory structures.") .addValidator(StandardValidators.BOOLEAN_VALIDATOR) .dynamic(true) .defaultValue("false") .build(); private final ComponentLog logger; private final ProcessContext ctx; private Session session; private ChannelSftp sftp; private boolean closed = false; private String homeDir; private final boolean disableDirectoryListing; public SFTPTransfer(final ProcessContext processContext, final ComponentLog logger) { this.ctx = processContext; this.logger = logger; final PropertyValue disableListing = processContext.getProperty(DISABLE_DIRECTORY_LISTING); disableDirectoryListing = disableListing == null ? false : Boolean.TRUE.equals(disableListing.asBoolean()); } @Override public String getProtocolName() { return "sftp"; } @Override public List<FileInfo> getListing() throws IOException { final String path = ctx.getProperty(FileTransfer.REMOTE_PATH).evaluateAttributeExpressions().getValue(); final int depth = 0; final int maxResults; final PropertyValue batchSizeValue = ctx.getProperty(FileTransfer.REMOTE_POLL_BATCH_SIZE); if (batchSizeValue == null) { maxResults = Integer.MAX_VALUE; } else { final Integer configuredValue = batchSizeValue.asInteger(); maxResults = configuredValue == null ? Integer.MAX_VALUE : configuredValue; } final List<FileInfo> listing = new ArrayList<>(1000); getListing(path, depth, maxResults, listing); return listing; } private void getListing(final String path, final int depth, final int maxResults, final List<FileInfo> listing) throws IOException { if (maxResults < 1 || listing.size() >= maxResults) { return; } if (depth >= 100) { logger.warn(this + " had to stop recursively searching directories at a recursive depth of " + depth + " to avoid memory issues"); return; } final boolean ignoreDottedFiles = ctx.getProperty(FileTransfer.IGNORE_DOTTED_FILES).asBoolean(); final boolean recurse = ctx.getProperty(FileTransfer.RECURSIVE_SEARCH).asBoolean(); final String fileFilterRegex = ctx.getProperty(FileTransfer.FILE_FILTER_REGEX).getValue(); final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex); final String pathFilterRegex = ctx.getProperty(FileTransfer.PATH_FILTER_REGEX).getValue(); final Pattern pathPattern = (!recurse || pathFilterRegex == null) ? null : Pattern.compile(pathFilterRegex); final String remotePath = ctx.getProperty(FileTransfer.REMOTE_PATH).evaluateAttributeExpressions().getValue(); // check if this directory path matches the PATH_FILTER_REGEX boolean pathFilterMatches = true; if (pathPattern != null) { Path reldir = path == null ? Paths.get(".") : Paths.get(path); if (remotePath != null) { reldir = Paths.get(remotePath).relativize(reldir); } if (reldir != null && !reldir.toString().isEmpty()) { if (!pathPattern.matcher(reldir.toString().replace("\\", "/")).matches()) { pathFilterMatches = false; } } } final ChannelSftp sftp = getChannel(null); final boolean isPathMatch = pathFilterMatches; final List<LsEntry> subDirs = new ArrayList<>(); try { final LsEntrySelector filter = new LsEntrySelector() { @Override public int select(final LsEntry entry) { final String entryFilename = entry.getFilename(); // skip over 'this directory' and 'parent directory' special // files regardless of ignoring dot files if (entryFilename.equals(".") || entryFilename.equals("..")) { return LsEntrySelector.CONTINUE; } // skip files and directories that begin with a dot if we're // ignoring them if (ignoreDottedFiles && entryFilename.startsWith(".")) { return LsEntrySelector.CONTINUE; } // if is a directory and we're supposed to recurse if (recurse && entry.getAttrs().isDir()) { subDirs.add(entry); return LsEntrySelector.CONTINUE; } // if is not a directory and is not a link and it matches // FILE_FILTER_REGEX - then let's add it if (!entry.getAttrs().isDir() && !entry.getAttrs().isLink() && isPathMatch) { if (pattern == null || pattern.matcher(entryFilename).matches()) { listing.add(newFileInfo(entry, path)); } } if (listing.size() >= maxResults) { return LsEntrySelector.BREAK; } return LsEntrySelector.CONTINUE; } }; if (path == null || path.trim().isEmpty()) { sftp.ls(".", filter); } else { sftp.ls(path, filter); } } catch (final SftpException e) { final String pathDesc = path == null ? "current directory" : path; switch (e.id) { case ChannelSftp.SSH_FX_NO_SUCH_FILE: throw new FileNotFoundException("Could not perform listing on " + pathDesc + " because could not find the file on the remote server"); case ChannelSftp.SSH_FX_PERMISSION_DENIED: throw new PermissionDeniedException("Could not perform listing on " + pathDesc + " due to insufficient permissions"); default: throw new IOException("Failed to obtain file listing for " + pathDesc, e); } } for (final LsEntry entry : subDirs) { final String entryFilename = entry.getFilename(); final File newFullPath = new File(path, entryFilename); final String newFullForwardPath = newFullPath.getPath().replace("\\", "/"); try { getListing(newFullForwardPath, depth + 1, maxResults, listing); } catch (final IOException e) { logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory"); } } } private FileInfo newFileInfo(final LsEntry entry, String path) { if (entry == null) { return null; } final File newFullPath = new File(path, entry.getFilename()); final String newFullForwardPath = newFullPath.getPath().replace("\\", "/"); String perms = entry.getAttrs().getPermissionsString(); if (perms.length() > 9) { perms = perms.substring(perms.length() - 9); } FileInfo.Builder builder = new FileInfo.Builder() .filename(entry.getFilename()) .fullPathFileName(newFullForwardPath) .directory(entry.getAttrs().isDir()) .size(entry.getAttrs().getSize()) .lastModifiedTime(entry.getAttrs().getMTime() * 1000L) .permissions(perms) .owner(Integer.toString(entry.getAttrs().getUId())) .group(Integer.toString(entry.getAttrs().getGId())); return builder.build(); } @Override public InputStream getInputStream(final String remoteFileName) throws IOException { return getInputStream(remoteFileName, null); } @Override public InputStream getInputStream(final String remoteFileName, final FlowFile flowFile) throws IOException { final ChannelSftp sftp = getChannel(flowFile); try { return sftp.get(remoteFileName); } catch (final SftpException e) { switch (e.id) { case ChannelSftp.SSH_FX_NO_SUCH_FILE: throw new FileNotFoundException("Could not find file " + remoteFileName + " on remote SFTP Server"); case ChannelSftp.SSH_FX_PERMISSION_DENIED: throw new PermissionDeniedException("Insufficient permissions to read file " + remoteFileName + " from remote SFTP Server", e); default: throw new IOException("Failed to obtain file content for " + remoteFileName, e); } } } @Override public void flush() throws IOException { // nothing needed here } @Override public void deleteFile(final String path, final String remoteFileName) throws IOException { final String fullPath = (path == null) ? remoteFileName : (path.endsWith("/")) ? path + remoteFileName : path + "/" + remoteFileName; try { sftp.rm(fullPath); } catch (final SftpException e) { switch (e.id) { case ChannelSftp.SSH_FX_NO_SUCH_FILE: throw new FileNotFoundException("Could not find file " + remoteFileName + " to remove from remote SFTP Server"); case ChannelSftp.SSH_FX_PERMISSION_DENIED: throw new PermissionDeniedException("Insufficient permissions to delete file " + remoteFileName + " from remote SFTP Server", e); default: throw new IOException("Failed to delete remote file " + fullPath, e); } } } @Override public void deleteDirectory(final String remoteDirectoryName) throws IOException { try { sftp.rm(remoteDirectoryName); } catch (final SftpException e) { throw new IOException("Failed to delete remote directory " + remoteDirectoryName, e); } } @Override public void ensureDirectoryExists(final FlowFile flowFile, final File directoryName) throws IOException { final ChannelSftp channel = getChannel(flowFile); final String remoteDirectory = directoryName.getAbsolutePath().replace("\\", "/").replaceAll("^.\\:", ""); // if we disable the directory listing, we just want to blindly perform the mkdir command, // eating any exceptions thrown (like if the directory already exists). if (disableDirectoryListing) { try { channel.mkdir(remoteDirectory); } catch (SftpException e) { if (e.id != ChannelSftp.SSH_FX_FAILURE) { throw new IOException("Could not blindly create remote directory due to " + e.getMessage(), e); } } return; } // end if disableDirectoryListing boolean exists; try { channel.stat(remoteDirectory); exists = true; } catch (final SftpException e) { if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { // No Such File exists = false; } else { throw new IOException("Failed to determine if remote directory exists at " + remoteDirectory + " due to " + e, e); } } if (!exists) { // first ensure parent directories exist before creating this one if (directoryName.getParent() != null && !directoryName.getParentFile().equals(new File(File.separator))) { ensureDirectoryExists(flowFile, directoryName.getParentFile()); } logger.debug("Remote Directory {} does not exist; creating it", new Object[] {remoteDirectory}); try { channel.mkdir(remoteDirectory); logger.debug("Created {}", new Object[] {remoteDirectory}); } catch (final SftpException e) { throw new IOException("Failed to create remote directory " + remoteDirectory + " due to " + e, e); } } } private ChannelSftp getChannel(final FlowFile flowFile) throws IOException { if (sftp != null) { String sessionhost = session.getHost(); String desthost = ctx.getProperty(HOSTNAME).evaluateAttributeExpressions(flowFile).getValue(); if (sessionhost.equals(desthost)) { // destination matches so we can keep our current session return sftp; } else { // this flowFile is going to a different destination, reset session close(); } } final JSch jsch = new JSch(); try { final Session session = jsch.getSession(ctx.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue(), ctx.getProperty(HOSTNAME).evaluateAttributeExpressions(flowFile).getValue(), ctx.getProperty(PORT).evaluateAttributeExpressions(flowFile).asInteger().intValue()); final String hostKeyVal = ctx.getProperty(HOST_KEY_FILE).getValue(); if (hostKeyVal != null) { jsch.setKnownHosts(hostKeyVal); } final Properties properties = new Properties(); properties.setProperty("StrictHostKeyChecking", ctx.getProperty(STRICT_HOST_KEY_CHECKING).asBoolean() ? "yes" : "no"); properties.setProperty("PreferredAuthentications", "publickey,password,keyboard-interactive"); final PropertyValue compressionValue = ctx.getProperty(FileTransfer.USE_COMPRESSION); if (compressionValue != null && "true".equalsIgnoreCase(compressionValue.getValue())) { properties.setProperty("compression.s2c", "zlib@openssh.com,zlib,none"); properties.setProperty("compression.c2s", "zlib@openssh.com,zlib,none"); } else { properties.setProperty("compression.s2c", "none"); properties.setProperty("compression.c2s", "none"); } session.setConfig(properties); final String privateKeyFile = ctx.getProperty(PRIVATE_KEY_PATH).evaluateAttributeExpressions(flowFile).getValue(); if (privateKeyFile != null) { jsch.addIdentity(privateKeyFile, ctx.getProperty(PRIVATE_KEY_PASSPHRASE).evaluateAttributeExpressions(flowFile).getValue()); } final String password = ctx.getProperty(FileTransfer.PASSWORD).evaluateAttributeExpressions(flowFile).getValue(); if (password != null) { session.setPassword(password); } session.setTimeout(ctx.getProperty(FileTransfer.CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); session.connect(); this.session = session; this.closed = false; sftp = (ChannelSftp) session.openChannel("sftp"); sftp.connect(); session.setTimeout(ctx.getProperty(FileTransfer.DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); if (!ctx.getProperty(USE_KEEPALIVE_ON_TIMEOUT).asBoolean()) { session.setServerAliveCountMax(0); // do not send keepalive message on SocketTimeoutException } this.homeDir = sftp.getHome(); return sftp; } catch (final SftpException | JSchException e) { throw new IOException("Failed to obtain connection to remote host due to " + e.toString(), e); } } @Override public String getHomeDirectory(final FlowFile flowFile) throws IOException { getChannel(flowFile); return this.homeDir; } @Override public void close() throws IOException { if (closed) { return; } closed = true; try { if (null != sftp) { sftp.exit(); } } catch (final Exception ex) { logger.warn("Failed to close ChannelSftp due to {}", new Object[] {ex.toString()}, ex); } sftp = null; try { if (null != session) { session.disconnect(); } } catch (final Exception ex) { logger.warn("Failed to close session due to {}", new Object[] {ex.toString()}, ex); } session = null; } @Override public boolean isClosed() { return closed; } @Override @SuppressWarnings("unchecked") public FileInfo getRemoteFileInfo(final FlowFile flowFile, final String path, String filename) throws IOException { final ChannelSftp sftp = getChannel(flowFile); final String fullPath; if (path == null) { fullPath = filename; int slashpos = filename.lastIndexOf('/'); if (slashpos >= 0 && !filename.endsWith("/")) { filename = filename.substring(slashpos + 1); } } else { fullPath = path + "/" + filename; } final Vector<LsEntry> vector; try { vector = sftp.ls(fullPath); } catch (final SftpException e) { // ls throws exception if filename is not present if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { return null; } else { throw new IOException("Failed to obtain file listing for " + fullPath, e); } } LsEntry matchingEntry = null; for (final LsEntry entry : vector) { if (entry.getFilename().equalsIgnoreCase(filename)) { matchingEntry = entry; break; } } return newFileInfo(matchingEntry, path); } @Override public String put(final FlowFile flowFile, final String path, final String filename, final InputStream content) throws IOException { final ChannelSftp sftp = getChannel(flowFile); // destination path + filename final String fullPath = (path == null) ? filename : (path.endsWith("/")) ? path + filename : path + "/" + filename; // temporary path + filename String tempFilename = ctx.getProperty(TEMP_FILENAME).evaluateAttributeExpressions(flowFile).getValue(); if (tempFilename == null) { final boolean dotRename = ctx.getProperty(DOT_RENAME).asBoolean(); tempFilename = dotRename ? "." + filename : filename; } final String tempPath = (path == null) ? tempFilename : (path.endsWith("/")) ? path + tempFilename : path + "/" + tempFilename; try { sftp.put(content, tempPath); } catch (final SftpException e) { throw new IOException("Unable to put content to " + fullPath + " due to " + e, e); } final String lastModifiedTime = ctx.getProperty(LAST_MODIFIED_TIME).evaluateAttributeExpressions(flowFile).getValue(); if (lastModifiedTime != null && !lastModifiedTime.trim().isEmpty()) { try { final DateFormat formatter = new SimpleDateFormat(FILE_MODIFY_DATE_ATTR_FORMAT, Locale.US); final Date fileModifyTime = formatter.parse(lastModifiedTime); int time = (int) (fileModifyTime.getTime() / 1000L); sftp.setMtime(tempPath, time); } catch (final Exception e) { logger.error("Failed to set lastModifiedTime on {} to {} due to {}", new Object[] {tempPath, lastModifiedTime, e}); } } final String permissions = ctx.getProperty(PERMISSIONS).evaluateAttributeExpressions(flowFile).getValue(); if (permissions != null && !permissions.trim().isEmpty()) { try { int perms = numberPermissions(permissions); if (perms >= 0) { sftp.chmod(perms, tempPath); } } catch (final Exception e) { logger.error("Failed to set permission on {} to {} due to {}", new Object[] {tempPath, permissions, e}); } } final String owner = ctx.getProperty(REMOTE_OWNER).evaluateAttributeExpressions(flowFile).getValue(); if (owner != null && !owner.trim().isEmpty()) { try { sftp.chown(Integer.parseInt(owner), tempPath); } catch (final Exception e) { logger.error("Failed to set owner on {} to {} due to {}", new Object[] {tempPath, owner, e}); } } final String group = ctx.getProperty(REMOTE_GROUP).evaluateAttributeExpressions(flowFile).getValue(); if (group != null && !group.trim().isEmpty()) { try { sftp.chgrp(Integer.parseInt(group), tempPath); } catch (final Exception e) { logger.error("Failed to set group on {} to {} due to {}", new Object[] {tempPath, group, e}); } } if (!filename.equals(tempFilename)) { try { sftp.rename(tempPath, fullPath); } catch (final SftpException e) { try { sftp.rm(tempPath); throw new IOException("Failed to rename dot-file to " + fullPath + " due to " + e, e); } catch (final SftpException e1) { throw new IOException("Failed to rename dot-file to " + fullPath + " and failed to delete it when attempting to clean up", e1); } } } return fullPath; } @Override public void rename(final String source, final String target) throws IOException { final ChannelSftp sftp = getChannel(null); try { sftp.rename(source, target); } catch (final SftpException e) { switch (e.id) { case ChannelSftp.SSH_FX_NO_SUCH_FILE: throw new FileNotFoundException(); case ChannelSftp.SSH_FX_PERMISSION_DENIED: throw new PermissionDeniedException("Could not rename remote file " + source + " to " + target + " due to insufficient permissions"); default: throw new IOException(e); } } } protected int numberPermissions(String perms) { int number = -1; final Pattern rwxPattern = Pattern.compile("^[rwx-]{9}$"); final Pattern numPattern = Pattern.compile("\\d+"); if (rwxPattern.matcher(perms).matches()) { number = 0; if (perms.charAt(0) == 'r') { number |= 0x100; } if (perms.charAt(1) == 'w') { number |= 0x80; } if (perms.charAt(2) == 'x') { number |= 0x40; } if (perms.charAt(3) == 'r') { number |= 0x20; } if (perms.charAt(4) == 'w') { number |= 0x10; } if (perms.charAt(5) == 'x') { number |= 0x8; } if (perms.charAt(6) == 'r') { number |= 0x4; } if (perms.charAt(7) == 'w') { number |= 0x2; } if (perms.charAt(8) == 'x') { number |= 0x1; } } else if (numPattern.matcher(perms).matches()) { try { number = Integer.parseInt(perms, 8); } catch (NumberFormatException ignore) { } } return number; } static { JSch.setLogger(new com.jcraft.jsch.Logger() { @Override public boolean isEnabled(int level) { return true; } @Override public void log(int level, String message) { LoggerFactory.getLogger(SFTPTransfer.class).debug("SFTP Log: {}", message); } }); } }