/*
* Copyright 2013-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.sftp.outbound;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import org.apache.commons.io.FileUtils;
import org.hamcrest.Matchers;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.file.FileHeaders;
import org.springframework.integration.file.remote.MessageSessionCallback;
import org.springframework.integration.file.remote.session.Session;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.sftp.SftpTestSupport;
import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessagingException;
import org.springframework.messaging.PollableChannel;
import org.springframework.messaging.support.GenericMessage;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.util.FileCopyUtils;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.ChannelSftp.LsEntry;
/**
* @author Artem Bilan
* @author Gary Russell
* @since 3.0
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
public class SftpServerOutboundTests extends SftpTestSupport {
@Autowired
private PollableChannel output;
@Autowired
private DirectChannel inboundGet;
@Autowired
private DirectChannel invalidDirExpression;
@Autowired
private DirectChannel inboundMGet;
@Autowired
private DirectChannel inboundMGetRecursive;
@Autowired
private DirectChannel inboundMGetRecursiveFiltered;
@Autowired
private DirectChannel inboundMPut;
@Autowired
private DirectChannel inboundMPutRecursive;
@Autowired
private DirectChannel inboundMPutRecursiveFiltered;
@Autowired
private SessionFactory<LsEntry> sessionFactory;
@Autowired
private DirectChannel appending;
@Autowired
private DirectChannel ignoring;
@Autowired
private DirectChannel failing;
@Autowired
private DirectChannel inboundGetStream;
@Autowired
private DirectChannel inboundCallback;
@Autowired
private Config config;
@Before
public void setup() {
this.config.targetLocalDirectoryName = getTargetLocalDirectoryName();
}
@Test
public void testInt2866LocalDirectoryExpressionGET() {
Session<?> session = this.sessionFactory.getSession();
String dir = "sftpSource/";
long modified = setModifiedOnSource1();
this.inboundGet.send(new GenericMessage<Object>(dir + " sftpSource1.txt"));
Message<?> result = this.output.receive(1000);
assertNotNull(result);
File localFile = (File) result.getPayload();
assertThat(localFile.getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"),
containsString(dir.toUpperCase()));
assertPreserved(modified, localFile);
dir = "sftpSource/subSftpSource/";
this.inboundGet.send(new GenericMessage<Object>(dir + "subSftpSource1.txt"));
result = this.output.receive(1000);
assertNotNull(result);
localFile = (File) result.getPayload();
assertThat(localFile.getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"),
containsString(dir.toUpperCase()));
Session<?> session2 = this.sessionFactory.getSession();
assertSame(TestUtils.getPropertyValue(session, "targetSession.jschSession"),
TestUtils.getPropertyValue(session2, "targetSession.jschSession"));
}
@Test
public void testInt2866InvalidLocalDirectoryExpression() {
try {
this.invalidDirExpression.send(new GenericMessage<Object>("sftpSource/ sftpSource1.txt"));
fail("Exception expected.");
}
catch (Exception e) {
Throwable cause = e.getCause();
assertNotNull(cause);
cause = cause.getCause();
assertThat(cause, Matchers.instanceOf(IllegalArgumentException.class));
assertThat(cause.getMessage(), Matchers.startsWith("Failed to make local directory"));
}
}
@Test
@SuppressWarnings("unchecked")
public void testInt2866LocalDirectoryExpressionMGET() {
String dir = "sftpSource/";
long modified = setModifiedOnSource1();
this.inboundMGet.send(new GenericMessage<Object>(dir + "*.txt"));
Message<?> result = this.output.receive(1000);
assertNotNull(result);
List<File> localFiles = (List<File>) result.getPayload();
assertThat(localFiles.size(), Matchers.greaterThan(0));
boolean assertedModified = false;
for (File file : localFiles) {
assertThat(file.getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"),
containsString(dir));
if (file.getPath().contains("localTarget1")) {
assertedModified = assertPreserved(modified, file);
}
}
assertTrue(assertedModified);
dir = "sftpSource/subSftpSource/";
this.inboundMGet.send(new GenericMessage<Object>(dir + "*.txt"));
result = this.output.receive(1000);
assertNotNull(result);
localFiles = (List<File>) result.getPayload();
assertThat(localFiles.size(), Matchers.greaterThan(0));
for (File file : localFiles) {
assertThat(file.getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"),
containsString(dir));
}
}
@Test
@SuppressWarnings("unchecked")
public void testInt3172LocalDirectoryExpressionMGETRecursive() throws IOException {
String dir = "sftpSource/";
long modified = setModifiedOnSource1();
File secondRemote = new File(getSourceRemoteDirectory(), "sftpSource2.txt");
secondRemote.setLastModified(System.currentTimeMillis() - 1_000_000);
this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
Message<?> result = this.output.receive(1000);
assertNotNull(result);
List<File> localFiles = (List<File>) result.getPayload();
assertEquals(3, localFiles.size());
boolean assertedModified = false;
for (File file : localFiles) {
assertThat(file.getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"),
containsString(dir));
if (file.getPath().contains("localTarget1")) {
assertedModified = assertPreserved(modified, file);
}
}
assertTrue(assertedModified);
assertThat(localFiles.get(2).getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"),
containsString(dir + "subSftpSource"));
File secondTarget = new File(getTargetLocalDirectory() + File.separator + "sftpSource", "localTarget2.txt");
ByteArrayOutputStream remoteContents = new ByteArrayOutputStream();
ByteArrayOutputStream localContents = new ByteArrayOutputStream();
FileUtils.copyFile(secondRemote, remoteContents);
FileUtils.copyFile(secondTarget, localContents);
String localAsString = new String(localContents.toByteArray());
assertEquals(new String(remoteContents.toByteArray()), localAsString);
long oldLastModified = secondRemote.lastModified();
FileUtils.copyInputStreamToFile(new ByteArrayInputStream("junk".getBytes()), secondRemote);
long newLastModified = secondRemote.lastModified();
secondRemote.setLastModified(oldLastModified);
this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
this.output.receive(0);
localContents = new ByteArrayOutputStream();
FileUtils.copyFile(secondTarget, localContents);
assertEquals(localAsString, new String(localContents.toByteArray()));
secondRemote.setLastModified(newLastModified);
this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
this.output.receive(0);
localContents = new ByteArrayOutputStream();
FileUtils.copyFile(secondTarget, localContents);
assertEquals("junk", new String(localContents.toByteArray()));
// restore the remote file contents
FileUtils.copyInputStreamToFile(new ByteArrayInputStream(localAsString.getBytes()), secondRemote);
}
private long setModifiedOnSource1() {
File firstRemote = new File(getSourceRemoteDirectory(), " sftpSource1.txt");
firstRemote.setLastModified(System.currentTimeMillis() - 1_000_000);
long modified = firstRemote.lastModified();
assertTrue(modified > 0);
return modified;
}
private boolean assertPreserved(long modified, File file) {
assertTrue("lastModified wrong by " + (modified - file.lastModified()),
Math.abs(file.lastModified() - modified) < 1_000);
return true;
}
@Test
@SuppressWarnings("unchecked")
public void testInt3172LocalDirectoryExpressionMGETRecursiveFiltered() {
String dir = "sftpSource/";
this.inboundMGetRecursiveFiltered.send(new GenericMessage<Object>(dir + "*"));
Message<?> result = this.output.receive(1000);
assertNotNull(result);
List<File> localFiles = (List<File>) result.getPayload();
// should have filtered sftpSource2.txt
assertEquals(2, localFiles.size());
for (File file : localFiles) {
assertThat(file.getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"),
containsString(dir));
}
assertThat(localFiles.get(1).getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"),
containsString(dir + "subSftpSource"));
}
/**
* Only runs with a real server (see class javadocs).
*/
@Test
public void testInt3100RawGET() throws Exception {
Session<?> session = this.sessionFactory.getSession();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
FileCopyUtils.copy(session.readRaw("sftpSource/ sftpSource1.txt"), baos);
assertTrue(session.finalizeRaw());
assertEquals("source1", new String(baos.toByteArray()));
baos = new ByteArrayOutputStream();
FileCopyUtils.copy(session.readRaw("sftpSource/sftpSource2.txt"), baos);
assertTrue(session.finalizeRaw());
assertEquals("source2", new String(baos.toByteArray()));
session.close();
}
@Test
public void testInt3047ConcurrentSharedSession() throws Exception {
final Session<?> session1 = this.sessionFactory.getSession();
final Session<?> session2 = this.sessionFactory.getSession();
final PipedInputStream pipe1 = new PipedInputStream();
PipedOutputStream out1 = new PipedOutputStream(pipe1);
final PipedInputStream pipe2 = new PipedInputStream();
PipedOutputStream out2 = new PipedOutputStream(pipe2);
final CountDownLatch latch1 = new CountDownLatch(1);
final CountDownLatch latch2 = new CountDownLatch(1);
Executors.newSingleThreadExecutor().execute(() -> {
try {
session1.write(pipe1, "foo.txt");
}
catch (IOException e) {
e.printStackTrace();
}
latch1.countDown();
});
Executors.newSingleThreadExecutor().execute(() -> {
try {
session2.write(pipe2, "bar.txt");
}
catch (IOException e) {
e.printStackTrace();
}
latch2.countDown();
});
out1.write('a');
out2.write('b');
out1.write('c');
out2.write('d');
out1.write('e');
out2.write('f');
out1.close();
out2.close();
assertTrue(latch1.await(10, TimeUnit.SECONDS));
assertTrue(latch2.await(10, TimeUnit.SECONDS));
ByteArrayOutputStream bos1 = new ByteArrayOutputStream();
ByteArrayOutputStream bos2 = new ByteArrayOutputStream();
session1.read("foo.txt", bos1);
session2.read("bar.txt", bos2);
assertEquals("ace", new String(bos1.toByteArray()));
assertEquals("bdf", new String(bos2.toByteArray()));
session1.remove("foo.txt");
session2.remove("bar.txt");
session1.close();
session2.close();
}
@Test
public void testInt3088MPutNotRecursive() throws Exception {
Session<?> session = sessionFactory.getSession();
session.close();
session = TestUtils.getPropertyValue(session, "targetSession", Session.class);
ChannelSftp channel = spy(TestUtils.getPropertyValue(session, "channel", ChannelSftp.class));
new DirectFieldAccessor(session).setPropertyValue("channel", channel);
String dir = "sftpSource/";
this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
while (output.receive(0) != null) {
// drain
}
this.inboundMPut.send(new GenericMessage<File>(getSourceLocalDirectory()));
@SuppressWarnings("unchecked")
Message<List<String>> out = (Message<List<String>>) this.output.receive(1000);
assertNotNull(out);
assertEquals(2, out.getPayload().size());
assertThat(out.getPayload().get(0),
not(equalTo(out.getPayload().get(1))));
assertThat(
out.getPayload().get(0),
anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt")));
assertThat(
out.getPayload().get(1),
anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt")));
verify(channel).chmod(384, "sftpTarget/localSource1.txt"); // 384 = 600 octal
verify(channel).chmod(384, "sftpTarget/localSource2.txt");
}
@Test
public void testInt3088MPutRecursive() {
String dir = "sftpSource/";
this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
while (output.receive(0) != null) {
// drain
}
this.inboundMPutRecursive.send(new GenericMessage<File>(getSourceLocalDirectory()));
@SuppressWarnings("unchecked")
Message<List<String>> out = (Message<List<String>>) this.output.receive(1000);
assertNotNull(out);
assertEquals(3, out.getPayload().size());
assertThat(out.getPayload().get(0),
not(equalTo(out.getPayload().get(1))));
assertThat(
out.getPayload().get(0),
anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt"),
equalTo("sftpTarget/subLocalSource/subLocalSource1.txt")));
assertThat(
out.getPayload().get(1),
anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt"),
equalTo("sftpTarget/subLocalSource/subLocalSource1.txt")));
assertThat(
out.getPayload().get(2),
anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt"),
equalTo("sftpTarget/subLocalSource/subLocalSource1.txt")));
}
@Test
public void testInt3088MPutRecursiveFiltered() {
String dir = "sftpSource/";
this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
while (output.receive(0) != null) {
// drain
}
this.inboundMPutRecursiveFiltered.send(new GenericMessage<File>(getSourceLocalDirectory()));
@SuppressWarnings("unchecked")
Message<List<String>> out = (Message<List<String>>) this.output.receive(1000);
assertNotNull(out);
assertEquals(2, out.getPayload().size());
assertThat(out.getPayload().get(0),
not(equalTo(out.getPayload().get(1))));
assertThat(
out.getPayload().get(0),
anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt"),
equalTo("sftpTarget/subLocalSource/subLocalSource1.txt")));
assertThat(
out.getPayload().get(1),
anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt"),
equalTo("sftpTarget/subLocalSource/subLocalSource1.txt")));
}
@Test
public void testInt3412FileMode() {
Message<String> m = MessageBuilder.withPayload("foo")
.setHeader(FileHeaders.FILENAME, "appending.txt")
.build();
appending.send(m);
appending.send(m);
SftpRemoteFileTemplate template = new SftpRemoteFileTemplate(sessionFactory);
assertLength6(template);
ignoring.send(m);
assertLength6(template);
try {
failing.send(m);
fail("Expected exception");
}
catch (MessagingException e) {
assertThat(e.getCause().getCause().getMessage(), containsString("The destination file already exists"));
}
}
@Test
public void testStream() {
Session<?> session = spy(this.sessionFactory.getSession());
session.close();
String dir = "sftpSource/";
this.inboundGetStream.send(new GenericMessage<Object>(dir + " sftpSource1.txt"));
Message<?> result = this.output.receive(1000);
assertNotNull(result);
assertEquals("source1", result.getPayload());
assertEquals("sftpSource/", result.getHeaders().get(FileHeaders.REMOTE_DIRECTORY));
assertEquals(" sftpSource1.txt", result.getHeaders().get(FileHeaders.REMOTE_FILE));
verify(session).close();
}
@Test
public void testMessageSessionCallback() {
this.inboundCallback.send(new GenericMessage<String>("foo"));
Message<?> receive = this.output.receive(10000);
assertNotNull(receive);
assertEquals("FOO", receive.getPayload());
}
private void assertLength6(SftpRemoteFileTemplate template) {
LsEntry[] files = template.execute(session -> session.list("sftpTarget/appending.txt"));
assertEquals(1, files.length);
assertEquals(6, files[0].getAttrs().getSize());
}
@SuppressWarnings("unused")
private static final class TestMessageSessionCallback
implements MessageSessionCallback<LsEntry, Object> {
@Override
public Object doInSession(Session<ChannelSftp.LsEntry> session, Message<?> requestMessage) throws IOException {
return ((String) requestMessage.getPayload()).toUpperCase();
}
}
public static class Config {
private volatile String targetLocalDirectoryName;
@Bean
public SessionFactory<LsEntry> sftpSessionFactory() {
return SftpServerOutboundTests.sessionFactory();
}
public String getTargetLocalDirectoryName() {
return this.targetLocalDirectoryName;
}
}
}