FileDownloader.java

/**
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.airavata.gfac.bes.utils;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Map;

import org.unigrids.services.atomic.types.GridFileType;
import org.unigrids.services.atomic.types.ProtocolType;

import de.fzj.unicore.uas.client.FileTransferClient;
import de.fzj.unicore.uas.client.StorageClient;
import de.fzj.unicore.uas.client.UFTPConstants;
import de.fzj.unicore.uas.client.UFTPFileTransferClient;
import de.fzj.unicore.uas.fts.FiletransferOptions.IMonitorable;
import de.fzj.unicore.uas.fts.FiletransferOptions.SupportsPartialRead;

/**
 * helper that exports remote files from a UNICORE Storage 
 * to the local client machine.<br/>
 * Simple wildcards ("*" and "?") and download of 
 * directories are supported.
 * 
 * TODO this should be refactored so the single-file download logic 
 * is separated from the wildcard/directory/provided outputStream logic
 * 
 * @author schuller
 */
public class FileDownloader extends FileTransferBase{

	private boolean showProgress=true;

	private boolean forceFileOnly=false;
	
	private OutputStream targetStream=null;
	
	public FileDownloader(String from, String to, Mode mode){
		this(from,to,mode,true);
	}
	
	public FileDownloader(String from, String to, Mode mode, boolean failOnError){
		this.to=to;
		this.from=from;
		this.mode=mode;
		this.failOnError=failOnError;
	}
	
	public void perform(StorageClient sms)throws Exception{
		boolean isWildcard=hasWildCards(from);
		boolean isDirectory=false;
		GridFileType gridSource=null;
		if(isWildcard){
			performWildCardExport(sms);
		}
		else {
			//check if source is a directory
			gridSource=sms.listProperties(from);
			isDirectory=gridSource.getIsDirectory();
			if(isDirectory){
				if(forceFileOnly){
					throw new IOException("Source is a directory");
				}
				performDirectoryExport(gridSource, new File(to), sms);
			}
			else{
				download(gridSource,new File(to),sms);
			}
		}	
	}
	
	protected void performDirectoryExport(GridFileType directory, File targetDirectory, StorageClient sms)throws Exception{
		if(!targetDirectory.exists()|| !targetDirectory.canWrite()){
			throw new IOException("Target directory <"+to+"> does not exist or is not writable!");
		}
		if(!targetDirectory.isDirectory()){
			throw new IOException("Target <"+to+"> is not a directory!");
		}
		GridFileType[]gridFiles=sms.listDirectory(directory.getPath());
		for(GridFileType file: gridFiles){
			if(file.getIsDirectory()){
				if(!recurse) {
					System.out.println("Skipping directory "+file.getPath());
					continue;
				}
				else{
					File newTargetDirectory=new File(targetDirectory,getName(file.getPath()));
					boolean success=newTargetDirectory.mkdirs();
					if(!success)throw new IOException("Can create directory: "+newTargetDirectory.getAbsolutePath());
					performDirectoryExport(file, newTargetDirectory, sms);
					continue;
				}
			}
			download(file, new File(targetDirectory,getName(file.getPath())), sms);
		}
	}
	
	protected void performWildCardExport(StorageClient sms)throws Exception{
		String dir=getDir(from);
		if(dir==null)dir="/";
		GridFileType[] files=sms.find(dir, false, from, false, null, null);
		File targetDir=targetStream==null?new File(to):null;
		if(targetStream==null){
			if(!targetDir.isDirectory())throw new IOException("Target is not a directory.");
		}
		for(GridFileType f: files){
			download(f, targetDir, sms);
		}
	}	
	
	private String getDir(String path){
		return new File(path).getParent();
	}
	
	private String getName(String path){
		return new File(path).getName();
	}
	
	/**
	 * download a single regular file
	 * 
	 * @param source - grid file descriptor
	 * @param localFile - local file or directory to write to
	 * @param sms
	 * @throws Exception
	 */
	private void download(GridFileType source, File localFile, StorageClient sms)throws Exception{
		if(source==null || source.getIsDirectory()){
			throw new IllegalStateException("Source="+source); 
		}
		
		OutputStream os=targetStream!=null?targetStream:null;
		FileTransferClient ftc=null;
		try{
			String path=source.getPath();
			if(targetStream==null){
				if(localFile.isDirectory()){
					localFile=new File(localFile,getName(source.getPath()));
				}
				if(mode.equals(Mode.nooverwrite) && localFile.exists()){
					System.out.println("File exists and creation mode was set to 'nooverwrite'.");
					return; 
				}
				System.out.println("Downloading remote file '"+sms.getUrl()+"#/"+path+"' -> "+localFile.getAbsolutePath());
				os=new FileOutputStream(localFile.getAbsolutePath(), mode.equals(Mode.append));
			}
			
			chosenProtocol=sms.findSupportedProtocol(preferredProtocols.toArray(new ProtocolType.Enum[preferredProtocols.size()]));
			Map<String,String>extraParameters=makeExtraParameters(chosenProtocol);
			ftc=sms.getExport(path,extraParameters,chosenProtocol);
			configure(ftc, extraParameters);
			System.out.println("DEB:File transfer URL : "+ftc.getUrl());
//			ProgressBar p=null;
			if(ftc instanceof IMonitorable  && showProgress){
				long size=ftc.getSourceFileSize();
				if(isRange()){
					size=getRangeSize();
				}
//				p=new ProgressBar(localFile.getName(),size,msg);
//				((IMonitorable) ftc).setProgressListener(p);
			}
			long startTime=System.currentTimeMillis();
			if(isRange()){
				if(!(ftc instanceof SupportsPartialRead)){
					throw new Exception("Byte range is defined but protocol does not allow " +
							"partial read! Please choose a different protocol!");
				}
				System.out.println("Byte range: "+startByte+" - "+(getRangeSize()>0?endByte:""));
				SupportsPartialRead pReader=(SupportsPartialRead)ftc;
				pReader.readPartial(startByte, endByte-startByte+1, os);
			}
			else{
				ftc.readAllData(os);
			}
//			if(p!=null){
//				p.finish();
//			}
			if(timing){
				long duration=System.currentTimeMillis()-startTime;
				double rate=(double)localFile.length()/(double)duration;
				System.out.println("Rate: " +rate+ " kB/sec.");
			}
			if(targetStream==null)copyProperties(source, localFile);
		}
		finally{
			try{ 
				if(targetStream==null && os!=null){
					os.close();
				}
			}catch(Exception ignored){}
			if(ftc!=null){
				try{
					ftc.destroy();
				}catch(Exception e1){
//					System.out.println("Could not destroy the filetransfer client",e1);
				}
			}
		}
	}

	/**
	 * if possible, copy the remote executable flag to the local file
	 * @throws Exception
	 */
	private void copyProperties(GridFileType source, File localFile)throws Exception{
		try{
			localFile.setExecutable(source.getPermissions().getExecutable());
		}
		catch(Exception ex){
			//TODO: logging
//			("Can't set 'executable' flag for "+localFile.getName(), ex);
		}
	}
	
	private void configure(FileTransferClient ftc, Map<String,String>params){
		if(ftc instanceof UFTPFileTransferClient){
			UFTPFileTransferClient u=(UFTPFileTransferClient)ftc;
			String secret=params.get(UFTPConstants.PARAM_SECRET);
			u.setSecret(secret);
		}
	}

	public void setShowProgress(boolean showProgress) {
		this.showProgress = showProgress;
	}

	public void setForceFileOnly(boolean forceFileOnly) {
		this.forceFileOnly = forceFileOnly;
	}

	public void setTargetStream(OutputStream targetStream) {
		this.targetStream = targetStream;
	}
	
}