/*
* Data Hub Service (DHuS) - For Space data distribution.
* Copyright (C) 2013,2014,2015,2016 GAEL Systems
*
* This file is part of DHuS software sources.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.gael.dhus.datastore.processing;
import com.google.common.io.Closer;
import fr.gael.dhus.database.object.MetadataIndex;
import fr.gael.dhus.database.object.Product;
import fr.gael.dhus.datastore.IncomingManager;
import fr.gael.dhus.datastore.exception.DataStoreException;
import fr.gael.dhus.datastore.scanner.AsynchronousLinkedList.Event;
import fr.gael.dhus.datastore.scanner.AsynchronousLinkedList.Listener;
import fr.gael.dhus.datastore.scanner.Scanner;
import fr.gael.dhus.datastore.scanner.ScannerFactory;
import fr.gael.dhus.datastore.scanner.URLExt;
import fr.gael.dhus.service.ProductService;
import fr.gael.dhus.system.config.ConfigurationManager;
import fr.gael.dhus.util.AsyncFileLock;
import fr.gael.dhus.util.JTSFootprintParser;
import fr.gael.dhus.util.MultipleDigestInputStream;
import fr.gael.dhus.util.MultipleDigestOutputStream;
import fr.gael.dhus.util.UnZip;
import fr.gael.drb.DrbAttribute;
import fr.gael.drb.DrbFactory;
import fr.gael.drb.DrbNode;
import fr.gael.drb.DrbSequence;
import fr.gael.drb.impl.DrbNodeImpl;
import fr.gael.drb.impl.ftp.Transfer;
import fr.gael.drb.impl.spi.DrbNodeSpi;
import fr.gael.drb.impl.xml.XmlWriter;
import fr.gael.drb.query.ExternalVariable;
import fr.gael.drb.query.Query;
import fr.gael.drb.value.Value;
import fr.gael.drbx.cortex.DrbCortexItemClass;
import fr.gael.drbx.image.ImageFactory;
import fr.gael.drbx.image.impl.sdi.SdiImageFactory;
import fr.gael.drbx.image.jai.RenderingFactory;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.io.FileExistsException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.geotools.gml2.GMLConfiguration;
import org.geotools.xml.Configuration;
import org.geotools.xml.Parser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.xml.sax.InputSource;
import javax.activation.MimeType;
import javax.activation.MimeTypeParseException;
import javax.imageio.ImageIO;
import javax.media.jai.RenderedImageList;
import java.awt.image.RenderedImage;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
/**
* Manages product processing.
*/
@Component
public class ProcessingManager
{
private static final Logger LOGGER = LogManager.getLogger(ProcessingManager.class);
private static final int EOF = -1;
private static final String METADATA_NAMESPACE = "http://www.gael.fr/dhus#";
private static final String PROPERTY_IDENTIFIER = "identifier";
private static final String PROPERTY_INGESTIONDATE = "ingestionDate";
private static final String PROPERTY_METADATA_EXTRACTOR =
"metadataExtractor";
private static final String MIME_PLAIN_TEXT = "plain/text";
private static final String MIME_APPLICATION_GML = "application/gml+xml";
private static final String SIZE_QUERY=loadResourceFile("size.xql");
@Autowired
private ProductService productService;
@Autowired
private ConfigurationManager cfgManager;
@Autowired
private IncomingManager incomingManager;
@Autowired
private ScannerFactory scannerFactory;
/**
* Process product to finalize its ingestion
*/
@Transactional
public Product process (Product product)
{
long allStart = System.currentTimeMillis ();
LOGGER.info ("* Ingestion started.");
long start = System.currentTimeMillis ();
LOGGER.info (" - Product transfer started");
URL transferPath = transfer (
product.getOrigin (),product.getPath ().toString ());
if (transferPath != null)
{
product.setPath (transferPath);
productService.update (product);
}
LOGGER.info (" - Product transfer done in " +
(System.currentTimeMillis () - start) + "ms.");
start = System.currentTimeMillis ();
LOGGER.info (" - Product information extraction started");
// Force the ingestion date after transfer
URL productPath = product.getPath ();
File productFile = new File (productPath.getPath ());
DrbNode productNode =
ProcessingUtils.getNodeFromPath (productPath.getPath ());
try
{
DrbCortexItemClass productClass;
try
{
productClass = ProcessingUtils.getClassFromNode (productNode);
}
catch (IOException e)
{
throw new UnsupportedOperationException (
"Cannot compute item class.", e);
}
if (!productFile.exists ())
throw new UnsupportedOperationException ("File not found (" +
productFile.getPath () + ").");
// Set the product size
product.setSize (size (productFile));
// Set the product itemClass
product.setItemClass (productClass.getOntClass ().getURI ());
// Set the product identifier
String identifier = extractIdentifier (productNode, productClass);
if (identifier != null)
{
LOGGER.debug ("Found product identifier " + identifier);
product.setIdentifier (identifier);
}
else
{
LOGGER.warn ("No defined identifier - using filename");
product.setIdentifier (productFile.getName ());
}
LOGGER.info (" - Product information extraction done in " +
(System.currentTimeMillis () - start) + "ms.");
// Extract images
start = System.currentTimeMillis ();
LOGGER.info (" - Product images extraction started");
product = extractImages (productNode, product);
LOGGER.info (" - Product images extraction done in " +
(System.currentTimeMillis () - start) + "ms.");
// Generate download File
start = System.currentTimeMillis ();
LOGGER.info (" - Product downloadable file creation started");
product = generateDownloadFile (product);
LOGGER.info (" - Product downloadable file creation done in " +
(System.currentTimeMillis () - start) + "ms.");
// Set the product indexes
start = System.currentTimeMillis ();
LOGGER.info (" - Product indexes and footprint extraction started");
List<MetadataIndex> indexes =
extractIndexes (productNode, productClass);
SimpleDateFormat df =
new SimpleDateFormat ("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
indexes.add (new MetadataIndex ("Identifier",
null, "", PROPERTY_IDENTIFIER, product.getIdentifier ()));
if (indexes == null || indexes.isEmpty ())
{
LOGGER.warn ("No index processed for product " + product.getPath ());
}
else
{
product.setIndexes (indexes);
boolean jtsValid = false;
Iterator<MetadataIndex> iterator = indexes.iterator ();
while (iterator.hasNext ())
{
MetadataIndex index = iterator.next ();
// Extracts queryable informations to be stored into database.
if (index.getQueryable() != null)
{
// Begin position ("sensing start" or "validity start")
if (index.getQueryable().equalsIgnoreCase ("beginposition"))
{
try
{
product.setContentStart (df.parse (index.getValue ()));
}
catch (ParseException e)
{
LOGGER.warn ("Cannot set correctly product " +
"'content start' from indexes", e);
}
}
else
// End position ("sensing stop" or "validity stop")
if (index.getQueryable().equalsIgnoreCase ("endposition"))
{
try
{
product.setContentEnd (df.parse (index.getValue ()));
}
catch (ParseException e)
{
LOGGER.warn ("Cannot set correctly product " +
"'content end' from indexes", e);
}
}
}
/**
* Extract the footprints according to its types (GML or JTS)
*/
if (index.getType() != null)
{
if (index.getType().equalsIgnoreCase("application/gml+xml"))
{
String gml_footprint = index.getValue ();
if ((gml_footprint != null) &&
checkGMLFootprint (gml_footprint))
{
product.setFootPrint (gml_footprint);
}
else
{
LOGGER.error ("Incorrect on empty footprint for product " +
product.getPath ());
}
}
// Should not have been application/wkt ?
else if (index.getType().equalsIgnoreCase("application/jts"))
{
String jts_footprint = index.getValue ();
String parsedFootprint = JTSFootprintParser.checkJTSFootprint (jts_footprint);
jtsValid = parsedFootprint != null;
if (jtsValid)
{
index.setValue (parsedFootprint);
}
else
if (jts_footprint != null)
{
// JTS footprint is wrong; remove the corrupted
// footprint.
iterator.remove ();
}
}
}
}
if (!jtsValid)
{
LOGGER.error ("JTS footprint not existing or not valid, " +
"removing GML footprint on " + product.getPath ());
product.setFootPrint (null);
}
}
Date ingestion_date = new Date ();
indexes.add (new MetadataIndex ("Ingestion Date",
null, "product", PROPERTY_INGESTIONDATE, df.format (ingestion_date)));
product.setIngestionDate (ingestion_date);
LOGGER.info (" - Product indexes and footprint extraction done in " +
(System.currentTimeMillis () - start) + "ms.");
product.setUpdated (new Date ());
product.setProcessed (true);
LOGGER.info ("* Ingestion done in " +
(System.currentTimeMillis () - allStart) + "ms.");
return product;
}
finally
{
closeNode (productNode);
}
}
private void closeNode (DrbNode node)
{
if (node instanceof DrbNodeImpl)
{
DrbNodeImpl.class.cast (node).close (false);
}
}
/**
* Check GML Footprint validity
*/
private boolean checkGMLFootprint (String footprint)
{
try
{
Configuration configuration = new GMLConfiguration ();
Parser parser = new Parser (configuration);
parser.parse (new InputSource (new StringReader (footprint)));
return true;
}
catch (Exception e)
{
LOGGER.error("Error in extracted footprint: " + e.getMessage());
return false;
}
}
/**
* Retrieve product identifier using its Drb node and class.
*/
private String extractIdentifier (DrbNode productNode,
DrbCortexItemClass productClass)
{
java.util.Collection<String> properties = null;
// Get all values of the metadata properties attached to the item
// class or any of its super-classes
properties =
productClass.listPropertyStrings (METADATA_NAMESPACE +
PROPERTY_IDENTIFIER, false);
// Return immediately if no property value were found
if (properties == null)
{
LOGGER.warn ("Item \"" + productClass.getLabel () +
"\" has no identifier defined.");
return null;
}
// retrieve the first extractor
String property = properties.iterator ().next ();
// Filter possible XML markup brackets that could have been encoded
// in a CDATA section
property = property.replaceAll ("<", "<");
property = property.replaceAll (">", ">");
// Create a query for the current metadata extractor
Query query = new Query (property);
// Evaluate the XQuery
DrbSequence sequence = query.evaluate (productNode);
// Check that something results from the evaluation: jump to next
// value otherwise
if ( (sequence == null) || (sequence.getLength () < 1))
{
return null;
}
String identifier = sequence.getItem (0).toString ();
return identifier;
}
/**
* Loads product images from Drb node and stores information inside the
* product before returning it
*/
private Product extractImages (DrbNode productNode, Product product)
{
if (ImageIO.getUseCache()) ImageIO.setUseCache(false);
if (!ImageFactory.isImage (productNode))
{
LOGGER.debug ("No Image.");
return product;
}
RenderedImageList input_list = null;
RenderedImage input_image = null;
try
{
input_list = ImageFactory.createImage (productNode);
input_image = RenderingFactory.createDefaultRendering(input_list);
}
catch (Exception e)
{
LOGGER.debug ("Cannot retrieve default rendering");
if (LOGGER.isDebugEnabled ())
{
LOGGER.debug ("Error occurs during rendered image reader", e);
}
if (input_list == null)
{
return product;
}
input_image = input_list;
}
if (input_image == null)
{
return product;
}
// Generate Quicklook
int quicklook_width = cfgManager.getProductConfiguration ()
.getQuicklookConfiguration ().getWidth ();
int quicklook_height = cfgManager.getProductConfiguration ()
.getQuicklookConfiguration ().getHeight ();
// Deprecated code: raise warn.
boolean quicklook_cutting = cfgManager.getProductConfiguration ()
.getQuicklookConfiguration ().isCutting ();
if (quicklook_cutting)
LOGGER.warn(
"Quicklook \"cutting\" parameter is deprecated, will be ignored.");
LOGGER.info ("Generating Quicklook " +
quicklook_width + "x" + quicklook_height + " from " +
input_image.getWidth() + "x" + input_image.getHeight ());
RenderedImage image = null;
try
{
image = ProcessingUtils.resizeImage(input_image, quicklook_width, quicklook_height);
}
catch (InconsistentImageScale e)
{
LOGGER.error("Cannot resize image: {}", e.getMessage());
SdiImageFactory.close(input_list);
return product;
}
// Manages the quicklook output
File image_directory = incomingManager.getNewIncomingPath ();
AsyncFileLock afl = null;
try
{
Path path = Paths.get (image_directory.getAbsolutePath(),
".lock-writing");
afl = new AsyncFileLock(path);
afl.obtain (900000);
}
catch (IOException | InterruptedException | TimeoutException e)
{
LOGGER.warn ("Cannot lock incoming directory - continuing without (" +
e.getMessage () +")");
}
String identifier = product.getIdentifier ();
File file = new File (image_directory, identifier + "-ql.jpg");
try
{
if (ImageIO.write(image, "jpg", file))
{
product.setQuicklookPath (file.getPath ());
product.setQuicklookSize (file.length ());
}
}
catch (IOException e)
{
LOGGER.error ("Cannot save quicklook.",e);
}
// Generate Thumbnail
int thumbnail_width = cfgManager.getProductConfiguration ()
.getThumbnailConfiguration ().getWidth ();
int thumbnail_height = cfgManager.getProductConfiguration ()
.getThumbnailConfiguration ().getHeight ();
LOGGER.info ("Generating Thumbnail " +
thumbnail_width + "x" + thumbnail_height + " from " +
input_image.getWidth() + "x" + input_image.getHeight () + " image.");
try
{
image = ProcessingUtils.resizeImage(input_image, thumbnail_width, thumbnail_height);
}
catch (InconsistentImageScale e)
{
LOGGER.error("Cannot resize image: {}", e.getMessage());
SdiImageFactory.close(input_list);
if (afl != null)
{
afl.close();
}
return product;
}
// Manages the thumbnail output
file = new File (image_directory, identifier + "-th.jpg");
try
{
if (ImageIO.write(image, "jpg", file))
{
product.setThumbnailPath (file.getPath ());
product.setThumbnailSize (file.length ());
}
}
catch (IOException e)
{
LOGGER.error ("Cannot save thumbnail.",e);
}
SdiImageFactory.close (input_list);
if (afl != null)
{
afl.close();
}
return product;
}
/**
* Retrieve product indexes using its Drb node and class.
*/
private List<MetadataIndex> extractIndexes (DrbNode productNode,
DrbCortexItemClass productClass)
{
java.util.Collection<String> properties = null;
// Get all values of the metadata properties attached to the item
// class or any of its super-classes
properties =
productClass.listPropertyStrings (METADATA_NAMESPACE +
PROPERTY_METADATA_EXTRACTOR, false);
// Return immediately if no property value were found
if (properties == null)
{
LOGGER.warn ("Item \"" + productClass.getLabel () +
"\" has no metadata defined.");
return null;
}
// Prepare the index structure.
List<MetadataIndex> indexes = new ArrayList<MetadataIndex> ();
// Loop among retrieved property values
for (String property : properties)
{
// Filter possible XML markup brackets that could have been encoded
// in a CDATA section
property = property.replaceAll ("<", "<");
property = property.replaceAll (">", ">");
/*
* property = property.replaceAll("\n", " "); // Replace eol by blank
* space property = property.replaceAll(" +", " "); // Remove
* contiguous blank spaces
*/
// Create a query for the current metadata extractor
Query metadataQuery = null;
try
{
metadataQuery = new Query (property);
}
catch (Exception e)
{
LOGGER.error("Cannot compile metadata extractor " +
"(set debug mode to see details)", e);
if (LOGGER.isDebugEnabled())
{
LOGGER.debug(property);
}
throw new RuntimeException("Cannot compile metadata extractor",e);
}
// Evaluate the XQuery
DrbSequence metadataSequence = metadataQuery.evaluate (productNode);
// Check that something results from the evaluation: jump to next
// value otherwise
if ( (metadataSequence == null) || (metadataSequence.getLength () < 1))
{
continue;
}
// Loop among results
for (int iitem = 0; iitem < metadataSequence.getLength (); iitem++)
{
// Get current metadata node
DrbNode n = (DrbNode) metadataSequence.getItem (iitem);
// Get name
DrbAttribute name_att = n.getAttribute ("name");
Value name_v = null;
if (name_att != null) name_v = name_att.getValue ();
String name = null;
if (name_v != null)
name = name_v.convertTo (Value.STRING_ID).toString ();
// get type
DrbAttribute type_att = n.getAttribute ("type");
Value type_v = null;
if (type_att != null)
type_v = type_att.getValue ();
else
type_v = new fr.gael.drb.value.String (MIME_PLAIN_TEXT);
String type = type_v.convertTo (Value.STRING_ID).toString ();
// get category
DrbAttribute cat_att = n.getAttribute ("category");
Value cat_v = null;
if (cat_att != null)
cat_v = cat_att.getValue ();
else
cat_v = new fr.gael.drb.value.String ("product");
String category = cat_v.convertTo (Value.STRING_ID).toString ();
// get category
DrbAttribute qry_att = n.getAttribute ("queryable");
String queryable = null;
if (qry_att != null)
{
Value qry_v = qry_att.getValue ();
if (qry_v != null)
queryable = qry_v.convertTo (Value.STRING_ID).toString ();
}
// Get value
String value = null;
if (MIME_APPLICATION_GML.equals (type) && n.hasChild ())
{
ByteArrayOutputStream out = new ByteArrayOutputStream ();
XmlWriter.writeXML (n.getFirstChild (), out);
value = out.toString ();
try
{
out.close ();
}
catch (IOException e)
{
LOGGER.warn ("Cannot close stream !", e);
}
}
else
// Case of "text/plain"
{
Value value_v = n.getValue ();
if (value_v != null)
{
value = value_v.convertTo (Value.STRING_ID).toString ();
value = value.trim ();
}
}
if ( (name != null) && (value != null))
{
MetadataIndex index = new MetadataIndex ();
index.setName (name);
try
{
index.setType (new MimeType (type).toString ());
}
catch (MimeTypeParseException e)
{
LOGGER.warn (
"Wrong metatdata extractor mime type in class \"" +
productClass.getLabel () + "\" for metadata called \"" +
name + "\".", e);
}
index.setCategory (category);
index.setValue (value);
index.setQueryable (queryable);
indexes.add (index);
}
else
{
String field_name = "";
if (name != null)
field_name = name;
else
if (queryable != null)
field_name = queryable;
else
if (category != null)
field_name = "of category " + category;
LOGGER.warn ("Nothing extracted for field " + field_name);
}
}
}
return indexes;
}
/**
* Calculate a file or a folder size. This method recursively browse product
* according to the supported item loaded by Drb.
*/
long drb_size (File file)
{
String variable_name = "product_path";
// Use Drb/XQuery to compute size.
Query query = new Query(SIZE_QUERY);
if (query.getEnvironment().containsExternalVariable(variable_name))
{
ExternalVariable[] extVariables = query.getExternalVariables();
// Set the external variables
for (int iext = 0; iext < extVariables.length; iext++)
{
ExternalVariable var = extVariables[iext];
String varName = var.getName();
if (varName.equals(variable_name))
{
// Set it a new value
var.setValue(new
fr.gael.drb.value.String(file.getAbsolutePath()));
}
}
}
else
throw new UnsupportedOperationException ("Cannot set \"" +
variable_name + "\" XQuery parameter.");
DrbSequence sequence = query.evaluate(DrbFactory.openURI("."));
return ((fr.gael.drb.value.UnsignedLong)sequence.getItem(0).getValue().
convertTo(Value.UNSIGNED_LONG_ID)).longValue();
}
long system_size (File file)
{
long size=0;
if (file.isDirectory())
{
for (File subFile : file.listFiles ())
{
size += system_size (subFile);
}
}
else
{
size = file.length ();
}
return size;
}
long size (File file)
{
try
{
return drb_size(file);
}
catch (Exception e)
{
LOGGER.warn ("Cannot compute size via Drb API, using system API(" +
e.getMessage() + ").");
return system_size(file);
}
}
/**********************/
/** Product Transfer **/
/**********************/
/**
* Transfers product and stores information inside the
* product before returning it
*/
private URL transfer (String productOrigin, String productPath)
{
if (productOrigin == null)
{
return null;
}
if ( !productPath.equals (productOrigin))
{
return null;
}
File dest = incomingManager.getNewProductIncomingPath ();
AsyncFileLock afl = null;
try
{
Path path = Paths.get(dest.getParentFile().getAbsolutePath(),
".lock-writing");
afl = new AsyncFileLock(path);
afl.obtain (900000);
}
catch (IOException | InterruptedException | TimeoutException e)
{
LOGGER.warn ("Cannot lock incoming directory - continuing without (" +
e.getMessage () +")");
}
try
{
URL u = new URL (productOrigin);
String userInfos = u.getUserInfo ();
String username = null;
String password = null;
if (userInfos != null)
{
String[] infos = userInfos.split (":");
username = infos[0];
password = infos[1];
}
// Hooks to remove the partially transfered product
Hook hook = new Hook (dest.getParentFile ());
Runtime.getRuntime ().addShutdownHook (hook);
upload (productOrigin, username, password, dest);
Runtime.getRuntime ().removeShutdownHook (hook);
String local_filename = productOrigin;
if (productOrigin.endsWith ("/"))
{
local_filename =
local_filename.substring (0, local_filename.length () - 1);
}
local_filename =
local_filename.substring (local_filename.lastIndexOf ('/'));
File productFile = new File (dest, local_filename);
return productFile.toURI ().toURL ();
}
catch (Exception e)
{
FileUtils.deleteQuietly (dest);
throw new DataStoreException ("Cannot transfer product \"" +
productOrigin + "\".", e);
}
finally
{
if (afl != null)
{
afl.close();
}
}
}
private void upload (String url, final String username,
final String password, final File dest)
{
String remote_base_dir;
try
{
remote_base_dir = (new URL (url)).getPath ();
}
catch (MalformedURLException e1)
{
LOGGER.error ("Problem during upload", e1);
return;
}
final String remoteBaseDir = remote_base_dir;
Scanner scanner =
scannerFactory.getScanner (url, username, password, null);
// Get all files supported
scanner.setUserPattern (".*");
scanner.setForceNavigate (true);
scanner.getScanList ().addListener (new Listener<URLExt> ()
{
@Override
public void addedElement (Event<URLExt> e)
{
URLExt element = e.getElement ();
String remote_path = element.getUrl ().getPath ();
String remoteBase = remoteBaseDir;
if (remoteBase.endsWith ("/"))
{
remoteBase = remoteBase.substring (0, remoteBase.length () - 1);
}
String local_path_dir =
remote_path.replaceFirst (
remoteBase.substring (0, remoteBase.lastIndexOf ("/") + 1),
"");
File local_path = new File (dest, local_path_dir);
if ( !local_path.getParentFile ().exists ())
{
LOGGER.info ("Creating directory \"" +
local_path.getParentFile ().getPath () + "\".");
local_path.getParentFile ().mkdirs ();
local_path.getParentFile ().setWritable (true);
}
BufferedInputStream bis = null;
InputStream is = null;
FileOutputStream fos = null;
BufferedOutputStream bos = null;
int retry = 3;
boolean source_remove =
cfgManager.getFileScannersCronConfiguration ().isSourceRemove ();
if ( !element.isDirectory ())
{
DrbNode node =
DrbFactory.openURI (element.getUrl ().toExternalForm ());
long start = System.currentTimeMillis ();
do
{
try
{
LOGGER.info ("Transfering remote file \"" + remote_path +
"\" into \"" + local_path + "\".");
if ( (node instanceof DrbNodeSpi) &&
( ((DrbNodeSpi) node).hasImpl (File.class)))
{
File source =
(File) ((DrbNodeSpi) node).getImpl (File.class);
{
if (source_remove)
moveFile (source, local_path);
else
copyFile (source, local_path);
}
}
else
// Case of Use Transfer class to run
if ( (node instanceof DrbNodeSpi) &&
( ((DrbNodeSpi) node).hasImpl (Transfer.class)))
{
fos = new FileOutputStream (local_path);
bos = new BufferedOutputStream (fos);
Transfer t =
(Transfer) ((DrbNodeSpi) node)
.getImpl (Transfer.class);
t.copy (bos);
try
{
if (cfgManager
.getFileScannersCronConfiguration ()
.isSourceRemove ()) t.remove ();
}
catch (IOException ioe)
{
LOGGER.error (
"Unable to remove " + local_path.getPath (),
ioe);
}
}
else
{
if ((node instanceof DrbNodeSpi) &&
(((DrbNodeSpi) node).hasImpl (InputStream.class)))
{
is = (InputStream) ((DrbNodeSpi) node).
getImpl (InputStream.class);
}
else
is = element.getUrl ().openStream ();
bis = new BufferedInputStream (is);
fos = new FileOutputStream (local_path);
bos = new BufferedOutputStream (fos);
IOUtils.copyLarge (bis, bos);
}
// Prepare message
long stop = System.currentTimeMillis ();
long delay_ms = stop - start;
long size = local_path.length ();
String message = " in " + delay_ms + "ms";
if ( (size > 0) && (delay_ms > 0))
message += " at " +
((size/(1024*1024))/((float)delay_ms/1000.0))+"MB/s";
LOGGER.info ("Copy of " + node.getName () + " completed" +
message);
retry = 0;
}
catch (Exception excp)
{
if ( (retry - 1) <= 0)
{
LOGGER.error ("Cannot copy " + node.getName () +
" aborted.");
throw new RuntimeException ("Transfer Aborted.", excp);
}
else
{
LOGGER.warn ("Cannot copy " + node.getName () +
" retrying... (" + excp.getMessage () + ")");
try
{
Thread.sleep (1000);
}
catch (InterruptedException e1)
{
// Do nothing.
}
}
}
finally
{
try
{
if (bos != null) bos.close ();
if (fos != null) fos.close ();
if (bis != null) bis.close ();
if (is != null) is.close ();
}
catch (IOException exp)
{
LOGGER.error ("Error while closing copy streams.");
}
}
}
while (--retry > 0);
}
else
{
if ( !local_path.exists ())
{
LOGGER.info ("Creating directory \"" + local_path.getPath () +
"\".");
local_path.mkdirs ();
local_path.setWritable (true);
}
return;
}
}
@Override
public void removedElement (Event<URLExt> e)
{
}
});
try
{
scanner.scan ();
// Remove root product if required.
if (cfgManager.getFileScannersCronConfiguration ().isSourceRemove ())
{
try
{
DrbNode node = DrbFactory.openURI (url);
if (node instanceof DrbNodeSpi)
{
DrbNodeSpi spi = (DrbNodeSpi) node;
if (spi.hasImpl (File.class))
{
FileUtils.deleteQuietly ((File) spi.getImpl (File.class));
}
else
if (spi.hasImpl (Transfer.class))
{
((Transfer) spi.getImpl (Transfer.class)).remove ();
}
else
{
LOGGER.error ("Root product note removed (TBC)");
}
}
}
catch (Exception e)
{
LOGGER.warn ("Cannot remove input source (" + e.getMessage () +
").");
}
}
}
catch (Exception e)
{
if (e instanceof InterruptedException)
LOGGER.error ("Process interrupted by user");
else
LOGGER.error ("Error while uploading product", e);
// If something get wrong during upload: do not keep any residual
// data locally.
LOGGER.warn ("Remove residual uploaded data :" + dest.getPath ());
FileUtils.deleteQuietly (dest);
throw new UnsupportedOperationException ("Error during scan.", e);
}
}
private void copyFile (File source, File dest)
throws IOException, NoSuchAlgorithmException
{
String[] algorithms =
cfgManager.getDownloadConfiguration ().getChecksumAlgorithms ()
.split (",");
FileInputStream fis = null;
FileOutputStream fos = null;
MultipleDigestInputStream dis = null;
try
{
fis = new FileInputStream (source);
fos = new FileOutputStream (dest);
Boolean compute_checksum = UnZip.supported ( dest.getPath ());
if (compute_checksum)
{
dis = new MultipleDigestInputStream (fis, algorithms);
IOUtils.copyLarge (dis, fos);
// Write the checksums if any
for (String algorithm : algorithms)
{
String chk = dis.getMessageDigestAsHexadecimalString (algorithm);
FileUtils.write (new File (dest.getPath () + "." + algorithm),
chk);
}
}
else
IOUtils.copyLarge (fis, fos);
}
finally
{
IOUtils.closeQuietly (fos);
IOUtils.closeQuietly (dis);
IOUtils.closeQuietly (fis);
}
if (source.length () != dest.length ())
{
throw new IOException ("Failed to copy full contents from '" + source +
"' to '" + dest + "'");
}
}
private void moveFile (File src_file, File dest_file)
throws IOException, NoSuchAlgorithmException
{
if (src_file == null)
{
throw new NullPointerException ("Source must not be null");
}
if (dest_file == null)
{
throw new NullPointerException ("Destination must not be null");
}
if ( !src_file.exists ())
{
throw new FileNotFoundException ("Source '" + src_file +
"' does not exist");
}
if (src_file.isDirectory ())
{
throw new IOException ("Source '" + src_file + "' is a directory");
}
if (dest_file.exists ())
{
throw new FileExistsException ("Destination '" + dest_file +
"' already exists");
}
if (dest_file.isDirectory ())
{
throw new IOException ("Destination '" + dest_file +
"' is a directory");
}
boolean rename = src_file.renameTo (dest_file);
if ( !rename)
{
copyFile (src_file, dest_file);
if ( !src_file.delete ())
{
FileUtils.deleteQuietly (dest_file);
throw new IOException ("Failed to delete original file '" +
src_file + "' after copy to '" + dest_file + "'");
}
}
}
/**
* Shutdown hook used to manage incomplete transfer of products
*/
private class Hook extends Thread
{
private File path;
public Hook (File path)
{
this.path = path;
}
public void run ()
{
LOGGER.error ("Interruption during transfert to " + this.path);
FileUtils.deleteQuietly (path);
}
}
/************************************/
/** Generate Product Download File **/
/************************************/
/**
* Generates download file and stores information inside the
* product before returning it
*/
private Product generateDownloadFile (final Product product)
{
String product_id = product.getIdentifier ();
Map<String, String> checksums = null;
String[] algorithms =
cfgManager.getDownloadConfiguration ().getChecksumAlgorithms ()
.split (",");
if (product_id == null)
throw new NullPointerException ("Product \"" + product.getPath () +
"\" identifier not initialized.");
String product_path = product.getPath ().getPath ();
if (UnZip.supported (product_path))
{
product.setDownloadablePath (product_path);
product.setDownloadableSize (new File (product_path).length ());
}
File zip_file = null;
String zip_file_string = product.getDownloadablePath ();
if ( (zip_file_string == null) ||
( ! (new File (zip_file_string).exists ())))
{
File incoming = incomingManager.getNewIncomingPath ();
AsyncFileLock afl = null;
try
{
Path path = Paths.get(incoming.getAbsolutePath(), ".lock-writing");
afl = new AsyncFileLock(path);
afl.obtain (900000);
}
catch (IOException | InterruptedException | TimeoutException e)
{
LOGGER.warn ("Cannot lock incoming directory - " +
"continuing without (" + e.getMessage () +")");
}
zip_file = new File (incoming, (product_id + ".zip"));
LOGGER.info (zip_file.getName () +
": Generating zip file and its checksum.");
zip_file_string = zip_file.getPath ();
try
{
long start = System.currentTimeMillis ();
LOGGER.info ("Creation of downloadable archive into " +
zip_file_string);
checksums = processZip (product.getPath ().getPath (), zip_file);
long delay_ms = System.currentTimeMillis () - start;
long size_read =
new File (product.getPath ().getPath ()).length () /
(1024 * 1024);
long size_write = zip_file.length () / (1024 * 1024);
String message =
" in " + delay_ms + "ms. Read " + size_read + "MB, Write " +
size_write + "MB at " +
(size_write / ((float) (delay_ms + 1) / 1000)) + "MB/s";
LOGGER.info ("Downloadable archive saved (" +
product.getPath ().getFile () + ")" + message);
}
catch (IOException e)
{
LOGGER.error ("Cannot generate Zip archive for product \"" +
product.getPath () + "\".", e);
}
finally
{
afl.close ();
}
product.setDownloadablePath (zip_file_string);
product.setDownloadableSize (zip_file.length ());
}
else
{
try
{
if ( (checksums = findLocalChecksum (zip_file_string)) == null)
{
long start = System.currentTimeMillis ();
LOGGER.info (new File (zip_file_string).getName () +
": Computing checksum only.");
checksums = processChecksum (zip_file_string, algorithms);
/* Compute the output message */
long delay_ms = System.currentTimeMillis () - start;
long size = new File (zip_file_string).length () / (1024 * 1024);
String message =
" in " + delay_ms + "ms. Read " + size + "MB at " +
(size / ((float) (delay_ms + 1) / 1000)) + "MB/s";
LOGGER.info ("Checksum processed " + message);
}
else
{
LOGGER.info (new File (zip_file_string).getName () +
": Checksum retrieved from transfert.");
}
}
catch (Exception ioe)
{
LOGGER.warn ("cannot compute checksum.", ioe);
}
}
if (checksums != null)
{
product.getDownload ().getChecksums ().clear ();
product.getDownload ().getChecksums ().putAll (checksums);
}
return product;
}
private Map<String, String> processChecksum (String inpath,
String[] algorithms) throws IOException, NoSuchAlgorithmException
{
InputStream is = null;
MultipleDigestInputStream dis = null;
try
{
is = new FileInputStream (inpath);
dis = new MultipleDigestInputStream (is, algorithms);
readAll (dis);
}
finally
{
try
{
dis.close ();
is.close ();
}
catch (Exception e)
{
LOGGER.error ("Exception raised during ZIP stream close", e);
}
}
Map<String, String> checksums = new HashMap<String, String> ();
for (String algorithm : algorithms)
{
String chk = dis.getMessageDigestAsHexadecimalString (algorithm);
if (chk != null) checksums.put (algorithm, chk);
}
return checksums;
}
/**
* Read all the bytes of a file without output.
*
* @param is input stream to read
* @return the number of bytes read
* @throws IOException
*/
private long readAll (InputStream is) throws IOException
{
long count = 0;
int n = 0;
byte[] buffer = new byte[1024 * 4];
while (EOF != (n = is.read (buffer)))
{
count += n;
}
return count;
}
/**
* Retrieve checksums files located in the parent of the passed file.
* checksum files are identified by their extension that must be the digest
* manifest algorithm(SHA-1, SHA-256, MD5 ...) that
*
* @param file
* @return
*/
private Map<String, String> findLocalChecksum (String file)
{
File fileObject = new File (file);
File[] checksum_files =
new File (fileObject.getParent ()).listFiles (new FilenameFilter ()
{
@Override
public boolean accept (File dir, String name)
{
String algo = name.substring (name.lastIndexOf ('.') + 1);
try
{
MessageDigest.getInstance (algo);
return true;
}
catch (NoSuchAlgorithmException e)
{
return false;
}
}
});
if ( (checksum_files == null) || (checksum_files.length == 0))
return null;
Map<String, String> checksums = new HashMap<> ();
for (File checksum_file : checksum_files)
{
String chk;
try
{
chk = FileUtils.readFileToString (checksum_file);
}
catch (IOException e)
{
LOGGER.error ("Cannot read checksum in file " +
checksum_file.getPath ());
// Something is wrong: stop it right now!
return null;
}
String algo =
checksum_file.getName ().substring (
checksum_file.getName ().lastIndexOf ('.') + 1);
checksums.put (algo, chk);
}
return checksums;
}
/**
* Creates a zip file at the specified path with the contents of the
* specified directory.
*
* @param Input directory path. The directory were is located directory to
* archive.
* @param The full path of the zip file.
* @return the checksum accordig to
* fr.gael.dhus.datastore.processing.impl.zip.digest variable.
* @throws IOException If anything goes wrong
*/
private Map<String, String> processZip (String inpath, File output)
throws IOException
{
// Retrieve configuration settings
String[] algorithms =
cfgManager.getDownloadConfiguration ().getChecksumAlgorithms ()
.split (",");
int compressionLevel =
cfgManager.getDownloadConfiguration ().getCompressionLevel ();
FileOutputStream fOut = null;
BufferedOutputStream bOut = null;
ZipArchiveOutputStream tOut = null;
MultipleDigestOutputStream dOut = null;
try
{
fOut = new FileOutputStream (output);
if ( (algorithms != null) && (algorithms.length > 0))
{
try
{
dOut = new MultipleDigestOutputStream (fOut, algorithms);
bOut = new BufferedOutputStream (dOut);
}
catch (NoSuchAlgorithmException e)
{
LOGGER.error ("Problem computing checksum algorithms.", e);
dOut = null;
bOut = new BufferedOutputStream (fOut);
}
}
else
bOut = new BufferedOutputStream (fOut);
tOut = new ZipArchiveOutputStream (bOut);
tOut.setLevel (compressionLevel);
addFileToZip (tOut, inpath, "");
}
finally
{
try
{
tOut.finish ();
tOut.close ();
bOut.close ();
if (dOut != null) dOut.close ();
fOut.close ();
}
catch (Exception e)
{
LOGGER.error ("Exception raised during ZIP stream close", e);
}
}
if (dOut != null)
{
Map<String, String> checksums = new HashMap<String, String> ();
for (String algorithm : algorithms)
{
String chk = dOut.getMessageDigestAsHexadecimalString (algorithm);
if (chk != null) checksums.put (algorithm, chk);
}
return checksums;
}
return null;
}
/**
* Creates a zip entry for the path specified with a name built from the base
* passed in and the file/directory name. If the path is a directory, a
* recursive call is made such that the full directory is added to the zip.
*
* @param z_out The zip file's output stream
* @param path The filesystem path of the file/directory being added
* @param base The base prefix to for the name of the zip file entry
* @throws IOException If anything goes wrong
*/
private void addFileToZip (ZipArchiveOutputStream z_out, String path,
String base) throws IOException
{
File f = new File (path);
String entryName = base + f.getName ();
ZipArchiveEntry zipEntry = new ZipArchiveEntry (f, entryName);
z_out.putArchiveEntry (zipEntry);
if (f.isFile ())
{
FileInputStream fInputStream = null;
try
{
fInputStream = new FileInputStream (f);
org.apache.commons.compress.utils.IOUtils.copy (fInputStream,
z_out, 65535);
z_out.closeArchiveEntry ();
}
finally
{
fInputStream.close ();
}
}
else
{
z_out.closeArchiveEntry ();
File[] children = f.listFiles ();
if (children != null)
{
for (File child : children)
{
LOGGER.debug ("ZIP Adding " + child.getName ());
addFileToZip (z_out, child.getAbsolutePath (), entryName + "/");
}
}
}
}
/**
* Inner method used to load small ASCII resources that can be stored
* into memory. Thios resource shall be store close to this class (same
* package folder).
* @param resource the resource to load.
* @return the ASCII content of the resource.
*/
private static String loadResourceFile (String resource)
{
Closer closer = Closer.create();
String contents=null;
try
{
InputStream is = closer.register (ProcessingManager.class.
getResourceAsStream(resource));
contents=IOUtils.toString(is);
}
catch (Throwable e)
{
throw new UnsupportedOperationException(
"Cannot retrieve resource \"" + resource + "\".",e);
}
finally
{
try
{
closer.close();
}
catch (IOException e) { ; }
}
return contents;
}
}