FtpService.java

package com.cadrlife.devsearch.agent.service.ftp;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.SocketException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.cadrlife.devsearch.agent.service.RepoService;
import com.cadrlife.devsearch.domain.Project;
import com.google.common.base.Suppliers;
import com.google.common.base.Throwables;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;

public class FtpService extends RepoService {
	private static final Logger LOG = LoggerFactory.getLogger(FtpService.class);
	private String host;
	private String username;
	private String password;
	private String path;
	
	public FtpService(String host, String username, String password, String path) {
		this.host = host;
		this.username = username;
		this.password = password;
		this.path = path;
	}
	public static final List <String> TYPES_TO_INDEX = Arrays.asList("java", "cfm", "cfinc", "cfc", "xml", "xsl", "xslt", "js", "css",
			"vm", "ftl", "html", "htm", "config", "launch", "cfg", "sql", "jsp", "bat", "cmd", "properties", "groovy", "gs", "gsp", "txt",
			"sh", "classpath", "xsd", "tld", "mf", "ini", "json", "dtd", "wsdl", "py", "vins", "dat", "gradle", "htc", "vbs", "udf",
			"haml", "template", "scala", "conf", "fbprefs", "parm", "pl", "pm", "csv", "settings", "setting", "php",
			// VMS filetypes
			"inc", "pli", "ds", "cxx", "hxx", "fdl", "com",
			"c", "h", "cpp",
			// SQL
			"pb", "ps", "vw", "con", "grt", "tab", "trg", "pks", "pkb", "grt", "ind", "snp", "syn", "dpd", "synonym", "table", "view",
			"lst", "sqs", "ddl", "sequence", "schema", "prc", "viw");

	public FTPFile[] listFiles() throws SocketException, IOException {
		FTPClient client = new FTPClient();
		try {
			connectClient(client);
			return client.listFiles();
		}
		finally {
			client.disconnect();
		}
	}

	private void connectClient(FTPClient client) throws SocketException, IOException {
		client.connect(host);
		if (!client.isConnected()) {
			throw new IOException("Unable to connect: " + client.getReplyString());
		}
		client.login(username, password);
		if (230 != client.getReplyCode()) {
			throw new IOException("Unable to login: " + client.getReplyString());
		}
		client.changeWorkingDirectory(path);
		if (250 != client.getReplyCode()) {
			throw new IOException("Unable to change to directory: " + client.getReplyString());
		}
	}

	@Override
	public List <Project> findAllProjects() {
		return findProjectsByName(getRepoName());
	}

	@Override
	public void updateProject(File rootFile, Project projectName) {
		Map<String, FTPFile> files = new HashMap<String, FTPFile>();
		LoadingCache <String,Integer> versions = CacheBuilder.newBuilder().build(CacheLoader.from(Suppliers.ofInstance(0)));
		try {
			for (FTPFile remoteFile : listFiles()) {
				if (remoteFile.isFile()) {
					String name = StringUtils.substringBeforeLast(remoteFile.getName(), ";");
					String type = StringUtils.substringAfterLast(name, ".").toLowerCase();
					// Weird bug : files that have a ^ in them likely were put into CMS via FTP/SCP
					 if (TYPES_TO_INDEX.contains(type) && !name.contains("^")) {
						 Integer version = Integer.parseInt(StringUtils.substringAfterLast(remoteFile.getName(), ";"));
						 if (version > versions.get(name)) {
							 versions.put(name, version);
							 files.put(name, remoteFile);
						 }
					 }
				}
				
			}
			for (Entry<String,FTPFile> e : files.entrySet()) {
				String name = e.getKey();
				FTPFile ftpFile = e.getValue();
				FTPClient client = new FTPClient();
				try {
					connectClient(client);
					InputStream fileStream = client.retrieveFileStream(name);
					if (null == fileStream) {
						LOG.warn("null stream for {}, bytes {}, reply: {}", ftpFile.getName(), ftpFile.getSize(), client.getReplyString());
					} else {
						LOG.debug("saving to {}",rootFile.toPath().resolve(name));
						saveStreamToFile(fileStream, rootFile.toPath().resolve(name).toFile());
					}
				}
				finally {
					client.disconnect();
				}
				
			}
		}
		catch (Exception e) {
			Throwables.propagate(e);
		}
	}

	@Override
	public String getSourceType() {
		return "ftp";
	}

}