RemoteFileToTmpTests.java

/*
 * Copyright 2014-2015 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.springframework.batch.integration.x;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.net.ftp.FTPFile;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameter;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.integration.file.remote.InputStreamCallback;
import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.file.remote.session.Session;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.MessagingException;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.util.Assert;
import org.springframework.util.FileCopyUtils;
import org.springframework.xd.dirt.integration.bus.BusTestUtils;
import org.springframework.xd.dirt.integration.bus.local.LocalMessageBus;


/**
 *
 * @author Gary Russell
 */
@ContextConfiguration
@TestPropertySource(properties = { "restartable = false", "partitionResultsTimeout = 3600000", "xd.config.home = file:../../config" })
@RunWith(SpringJUnit4ClassRunner.class)
public class RemoteFileToTmpTests {

	private static final String tmpDir = System.getProperty("java.io.tmpdir");

	@Autowired
	private JobLauncher launcher;

	@Autowired
	private Job job;

	@Autowired
	private SessionFactory<?> sessionFactory;

	@Autowired
	@Qualifier("stepExecutionRequests.output")
	private MessageChannel requestsOut;

	@Autowired
	@Qualifier("stepExecutionRequests.input")
	private MessageChannel requestsIn;

	@Autowired
	@Qualifier("stepExecutionReplies.output")
	private MessageChannel repliesOut;

	@Autowired
	@Qualifier("stepExecutionReplies.input")
	private MessageChannel repliesIn;

	private LocalMessageBus bus;

	@SuppressWarnings({ "unchecked", "rawtypes" })
	@Before
	public void setup() throws Exception {
		byte[] bytes = "foo".getBytes();
		Session session = mock(Session.class);
		when(this.sessionFactory.getSession()).thenReturn(session);
		when(session.readRaw("/foo/bar.txt")).thenReturn(new ByteArrayInputStream(bytes));
		when(session.readRaw("/foo/baz.txt")).thenReturn(new ByteArrayInputStream(bytes));
		when(session.finalizeRaw()).thenReturn(true);
		Object[] fileList = new FTPFile[2];
		FTPFile file = new FTPFile();
		file.setName("bar.txt");
		fileList[0] = file;
		file = new FTPFile();
		file.setName("baz.txt");
		fileList[1] = file;
		when(session.list("/foo/")).thenReturn(fileList);

		this.bus = new LocalMessageBus();
		this.bus.setApplicationContext(BusTestUtils.MOCK_AC);
		this.bus.bindRequestor("foo", this.requestsOut, this.repliesIn, null);
		this.bus.bindReplier("foo", this.requestsIn, this.repliesOut, null);
		this.bus.afterPropertiesSet();
	}

	@After
	public void tearDown() {
		this.bus.unbindConsumer("foo", this.requestsOut);
		this.bus.unbindConsumer("foo", this.requestsIn);
		this.bus.unbindConsumer("foo", this.repliesIn);
		this.bus.unbindConsumer("foo", this.repliesOut);
	}

	@Test
	public void testSimple() throws Exception {
		Map<String, JobParameter> params = new HashMap<String, JobParameter>();
		params.put("remoteDirectory", new JobParameter("/foo/"));
		params.put("hdfsDirectory", new JobParameter("/qux"));
		JobParameters parameters = new JobParameters(params);
		JobExecution execution = launcher.run(job, parameters);
		assertEquals(ExitStatus.COMPLETED, execution.getExitStatus());

		File file = new File(tmpDir, "_foo_bar.txt");
		assertTrue(file.exists());
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		FileCopyUtils.copy(new FileInputStream(file), baos);
		assertEquals("foo", new String(baos.toByteArray()));
		file.delete();

		file = new File(tmpDir, "_foo_baz.txt");
		assertTrue(file.exists());
		baos = new ByteArrayOutputStream();
		FileCopyUtils.copy(new FileInputStream(file), baos);
		assertEquals("foo", new String(baos.toByteArray()));
		file.delete();
	}

	/**
	 * Replaces Hadoop tasklet so we can just write to java.io.tmpdir instead.
	 */
	public static class RemoteFileToTmpDirTasklet implements Tasklet {

		private final Logger logger = LoggerFactory.getLogger(this.getClass());

		private final RemoteFileTemplate<?> template;

		@SuppressWarnings("rawtypes")
		public RemoteFileToTmpDirTasklet(RemoteFileTemplate template) {
			template.setFileNameExpression(new SpelExpressionParser().parseExpression("payload"));
			try {
				template.afterPropertiesSet();
			}
			catch (Exception e) {
				logger.error("Failed to initialize RemoteFileTemplate", e);
			}
			this.template = template;
		}

		@Override
		public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
			final String fileName = chunkContext.getStepContext().getStepExecution().getExecutionContext().getString(
					"filePath");
			Assert.notNull(fileName);
			boolean result = this.template.get(MessageBuilder.withPayload(fileName).build(), new InputStreamCallback() {

				@Override
				public void doWithInputStream(InputStream stream) throws IOException {
					// TODO hadoop
					FileOutputStream out = new FileOutputStream(new File(tmpDir,
							fileName.replaceAll("\\/", "_")));
					FileCopyUtils.copy(stream, out);
				}

			});
			if (!result) {
				throw new MessagingException("Error during file transfer");
			}
			else {
				return RepeatStatus.FINISHED;
			}
		}

	}

}