/*
* Data Hub Service (DHuS) - For Space data distribution.
* Copyright (C) 2013,2014,2015,2016 GAEL Systems
*
* This file is part of DHuS software sources.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.gael.dhus.service;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import fr.gael.dhus.database.dao.CollectionDao;
import fr.gael.dhus.database.dao.ProductDao;
import fr.gael.dhus.database.dao.UserDao;
import fr.gael.dhus.database.object.Collection;
import fr.gael.dhus.database.object.MetadataIndex;
import fr.gael.dhus.database.object.Product;
import fr.gael.dhus.database.object.User;
import fr.gael.dhus.datastore.DataStore;
import fr.gael.dhus.datastore.Destination;
import fr.gael.dhus.datastore.Ingester;
import fr.gael.dhus.datastore.exception.DataStoreAlreadyExistException;
import fr.gael.dhus.datastore.exception.DataStoreException;
import fr.gael.dhus.datastore.exception.DataStoreLocalArchiveNotExistingException;
import fr.gael.dhus.datastore.processing.fair.FairCallable;
import fr.gael.dhus.datastore.processing.fair.FairThreadPoolTaskExecutor;
import fr.gael.dhus.datastore.scanner.AsynchronousLinkedList;
import fr.gael.dhus.datastore.scanner.AsynchronousLinkedList.Event;
import fr.gael.dhus.datastore.scanner.AsynchronousLinkedList.Listener;
import fr.gael.dhus.datastore.scanner.FileScannerWrapper;
import fr.gael.dhus.datastore.scanner.Scanner;
import fr.gael.dhus.datastore.scanner.ScannerFactory;
import fr.gael.dhus.datastore.scanner.URLExt;
import fr.gael.dhus.search.SolrDao;
import fr.gael.dhus.spring.cache.IncrementCache;
import fr.gael.dhus.spring.context.ApplicationContextProvider;
import fr.gael.dhus.system.config.ConfigurationManager;
import fr.gael.drbx.cortex.DrbCortexItemClass;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.solr.client.solrj.SolrServerException;
import org.hibernate.Hibernate;
import org.hibernate.criterion.DetachedCriteria;
import org.hibernate.criterion.Projections;
import org.hibernate.criterion.Restrictions;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.cache.annotation.Caching;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
/**
* Product Service provides connected clients with a set of method
* to interact with it.
*/
@Service
public class ProductService extends WebService
{
private static final Logger LOGGER = LogManager.getLogger(ProductService.class);
private static final long DAY_MILLI = 24 * 60 * 60 * 1_000;
@Autowired
private ProductDao productDao;
@Autowired
private CollectionDao collectionDao;
@Autowired
private CollectionService collectionService;
@Autowired
private UserDao userDao;
@Autowired
private FairThreadPoolTaskExecutor taskExecutor;
@Autowired
private ScannerFactory scannerFactory;
/** Configuration (etc/dhus.xml). */
@Autowired
private ConfigurationManager cfgManager;
@Autowired
private SolrDao solrDao;
@Autowired
private DataStore<Product> dataStore;
@PreAuthorize ("hasRole('ROLE_DATA_MANAGER')")
public Iterator<Product> GetProducts(String filter, Long collection_id, int skip)
{
return systemGetProducts(filter, collection_id, skip);
}
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
public Iterator<Product> systemGetProducts(String filter, Long collection_id, int skip)
{
return productDao.scrollFiltered(filter, collection_id, skip);
}
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value = "product", key = "#id")
public Product systemGetProduct (Long id)
{
return productDao.read (id);
}
@PreAuthorize ("hasAnyRole('ROLE_DATA_MANAGER','ROLE_SEARCH')")
@Cacheable (value = "product", key = "#id")
public Product getProduct (Long id)
{
return systemGetProduct (id);
}
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
public Product getProductNoCache (Long id)
{
return productDao.read (id);
}
@PreAuthorize ("hasAnyRole('ROLE_DATA_MANAGER','ROLE_SEARCH')")
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value = "products", key = "#ids")
public List<Product> getProducts (List<Long> ids)
{
return productDao.read(ids);
}
/**
* Gets a {@link Product} by its {@code UUID} (Protected).
* @see #getProduct(java.lang.String)
* @param uuid UUID unique identifier
* @return a {@link Product} or {@code null}
*/
@PreAuthorize ("hasAnyRole('ROLE_DATA_MANAGER','ROLE_SEARCH')")
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value = "product", key = "#uuid")
public Product getProduct (String uuid)
{
return systemGetProduct (uuid);
}
/**
* Gets a {@link Product} by its {@code UUID} (Unprotected).
* @param uuid UUID unique identifier
* @return a {@link Product} or {@code null}
*/
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value = "product", key = "#uuid")
public Product systemGetProduct (String uuid)
{
return productDao.getProductByUuid (uuid);
}
@PreAuthorize ("hasRole('ROLE_DOWNLOAD')")
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value = "product", key = "#id")
public Product getProductToDownload (Long id)
{
// TODO remove method cause duplicated and not used
return productDao.read (id);
}
@PreAuthorize ("hasAnyRole('ROLE_DOWNLOAD','ROLE_SEARCH')")
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
public InputStream getProductQuickLook (Long id)
{
// TODO remove method cause not used
Product product = getProduct (id);
if (!product.getQuicklookFlag ()) return null;
try
{
return new FileInputStream (product.getQuicklookPath ());
}
catch (Exception e)
{
LOGGER.warn ("Cannot retrieve Quicklook from product id #" + id,e);
}
return null;
}
@PreAuthorize ("hasAnyRole('ROLE_DOWNLOAD','ROLE_SEARCH')")
public long getProductQuickLookContentLength (Long id)
{
// TODO remove method cause not used
return getProduct (id).getQuicklookSize ();
}
@PreAuthorize ("hasAnyRole('ROLE_DOWNLOAD','ROLE_SEARCH')")
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
public InputStream getProductThumbnail (Long id)
{
// TODO remove method cause not used
Product product = getProduct (id);
if (!product.getThumbnailFlag ()) return null;
try
{
return new FileInputStream (product.getThumbnailPath ());
}
catch (Exception e)
{
LOGGER.warn ("Cannot retrieve Thumbnail from product id #" + id,e);
}
return null;
}
@PreAuthorize ("hasAnyRole('ROLE_DOWNLOAD','ROLE_SEARCH')")
public long getProductThumbnailContentLength (Long id)
{
// TODO remove method cause not used
return getProduct (id).getThumbnailSize ();
}
/**
* Returns the number of product belonging to the given Collection.
* <p><b>This method requires roles ROLE_DATA_MANAGER | ROLE_SEARCH.</b>
* @param filter an optionnal `where` clause (without the "where" token).
* @param collection_uuid the `Id` of the parent collection.
* @return number of Products.
*/
@PreAuthorize ("hasAnyRole('ROLE_DATA_MANAGER','ROLE_SEARCH')")
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value = "product_count", key = "{#filter, #collection_uuid}")
public Integer count(String filter, String collection_uuid)
{
return productDao.count(filter, collection_uuid);
}
@PreAuthorize ("hasAnyRole('ROLE_DATA_MANAGER','ROLE_SEARCH')")
@Transactional (readOnly = true, propagation = Propagation.REQUIRED)
@Cacheable (value = "product_count", key = "{#filter, null}")
public Integer count(String filter)
{
return productDao.count(filter, null);
}
/**
* Deletes references and binaries of product.
* @param pid product ID.
*/
@Transactional (readOnly=false, propagation=Propagation.REQUIRED)
@IncrementCache (name = "product_count", key = "all", value = -1)
@Caching(evict = {
@CacheEvict (value = "indexes", key = "#pid"),
@CacheEvict (value = "product", key = "#pid"),
@CacheEvict (value = "products", allEntries = true)})
public void systemDeleteProduct (Long pid)
{
Product product = productDao.read (pid);
if (product == null)
{
throw new DataStoreException ("Product #" + pid +
" not found in the system.");
}
systemDeleteProduct (product, Destination.TRASH);
}
/**
* Deletes a product.
* @param product product to delete.
* @param destination destination of product backup.
*/
@Transactional
@IncrementCache (name = "product_count", key = "all", value = -1)
@Caching (evict = {
@CacheEvict (value = "product", key = "#product.id"),
@CacheEvict (value = "product", key = "#product.uuid"),
@CacheEvict (value = "indexes", key = "#product.id"),
@CacheEvict (value = "products", allEntries = true)})
public void systemDeleteProduct (Product product, Destination destination)
{
if (product == null)
{
throw new IllegalArgumentException ("Product should not be null");
}
if (product.getLocked ())
{
throw new DataStoreException ("Cannot delete product : " + product +
". This product is locked by the system");
}
long start = System.currentTimeMillis ();
try
{
try
{
solrDao.remove (product.getId ());
}
catch (NullPointerException | IllegalStateException e)
{
LOGGER.warn ("Solr not running !", e);
}
productDao.delete (product);
dataStore.remove (product, destination);
long time = System.currentTimeMillis () - start;
LOGGER.info ("Deletion of product '"+ product.getIdentifier () +
"' (" + product.getDownloadableSize () + " bytes) successful" +
" spent " + time + "ms");
}
catch (SolrServerException | IOException e)
{
LOGGER.error ("Deletion of product " + product + " failed", e);
LOGGER.warn ("Please delete manually this product");
}
}
@PreAuthorize ("hasRole('ROLE_DATA_MANAGER')")
@Transactional (readOnly=false, propagation=Propagation.REQUIRED)
@Caching (evict = {
@CacheEvict (value = "product", key = "#pid"),
@CacheEvict (value = "products", allEntries = true),
@CacheEvict (value = "indexes", key = "#pid")
})
@IncrementCache (name = "product_count", key = "all", value = -1)
public void deleteProduct(Long pid)
{
systemDeleteProduct (pid);
}
@PreAuthorize ("isAuthenticated ()")
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value="product", key="#uuid")
public Product getProduct (String uuid, User u)
{
return productDao.getProductByUuid (uuid);
}
@PreAuthorize ("isAuthenticated ()")
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value="product_count", key="'all'")
public int count ()
{
return productDao.count ();
}
public boolean hasAccessToProduct (long user_id, long product_id)
{
return true;
}
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value = {"indexes"}, key = "#product_id")
public List<MetadataIndex> getIndexes(Long product_id)
{
Product product = productDao.read (product_id);
if (product == null) return new ArrayList<MetadataIndex> ();
Hibernate.initialize (product.getIndexes ());
return product.getIndexes ();
}
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
public Product getProductWithIndexes(Long product_id)
{
Product product = productDao.read (product_id);
if (product == null) return null;
Hibernate.initialize (product.getIndexes ());
return product;
}
@Transactional (readOnly=false, propagation=Propagation.REQUIRED)
@CacheEvict (value = {"indexes"}, key = "#product_id")
public void setIndexes(Long product_id, List<MetadataIndex>indexes)
{
Product product = productDao.read (product_id);
product.setIndexes (indexes);
productDao.update (product);
}
/**
* Adds a product in the database, the given product will not be queued for
* processing nor it will be submitted to the search engine.
* @param product a product to store in the database.
* @return the created product.
* @throws IllegalArgumentException incomplete products are not allowed.
*/
@Transactional (readOnly = false, propagation = Propagation.REQUIRED)
@Caching (evict = {
@CacheEvict (value = "product", allEntries = true),
@CacheEvict (value = "products", allEntries = true) })
@IncrementCache (name = "product_count", key = "all", value = 1)
public Product addProduct (Product product) throws IllegalArgumentException
{
URL path = product.getPath ();
String origin = product.getOrigin ();
if (path == null || origin == null || origin.isEmpty ())
{
throw new IllegalArgumentException ("product must have a path and an origin");
}
// FIXME do I have to check every field? isn't it done by hibernate based on column constraints?
Product final_product = this.productDao.create (product);
return final_product;
}
@Caching(evict = {
@CacheEvict(value = "product" , allEntries = true),
@CacheEvict(value = "products", allEntries = true)})
@Transactional
public Product addProduct (URL path, User owner, String origin)
throws DataStoreAlreadyExistException
{
if (productDao.exists (path))
{
throw new DataStoreAlreadyExistException ("Product \"" +
path.toExternalForm () + "\" already present in the system.");
}
/* **** CRITICAL SECTION *** */
/** THIS SECTION SHALL NEVER BE STOPPED BY CNTRL-C OR OTHER SIGNALS */
/* TODO: check if shutdownHook can protect this section */
Product product = new Product ();
product.setPath (path);
product.setOrigin (origin);
List<User> users = new ArrayList<User> ();
if (owner != null)
{
product.setOwner (owner);
users.add (userDao.read (owner.getUUID ()));
product.setAuthorizedUsers (new HashSet<User> (users));
}
product = productDao.create (product);
return product;
}
/**
* Process given unprocessed product.
* @param product to process.
* @param owner user owning that product.
* @param collections containing that product.
* @param scanner
* @param wrapper
* @return A future to get notified for the end of the processing.
* {@code get()} will return null, see {@link ProcessingCallable#call()}.
* May return {@code null} if a RejectedExecutionException has been thrown.
*/
public Future<Object> processProduct(Product product, User owner,
List<Collection>collections, Scanner scanner,
FileScannerWrapper wrapper)
{
Future<Object> future = null;
int retry = 10;
while (retry > 0)
{
try
{
ProcessingCallable pr = new ProcessingCallable(product, owner,
collections, scanner, wrapper);
future = taskExecutor.submit(pr);
retry = 0;
}
catch (RejectedExecutionException ree)
{
retry--;
if (retry <= 0) throw ree;
try
{
Thread.sleep (500);
}
catch (InterruptedException e)
{
LOGGER.warn ("Current thread has interrupted by another!", e);
}
}
}
return future;
}
/**
* Odata dedicated Services
*/
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value="products", key="{#criteria, #uuid, #skip, #top}")
public List<Product> getProducts (DetachedCriteria criteria, String uuid,
int skip, int top)
{
if (criteria == null)
{
criteria = DetachedCriteria.forClass (Product.class);
}
// get only processed products
criteria.add (Restrictions.eq ("processed", true));
if (uuid != null)
{
Collection collection = collectionDao.read (uuid);
if (collection != null)
{
criteria.add (Restrictions.in (
"id", collectionService.getProductIds (uuid)));
}
}
return productDao.listCriteria (criteria, skip, top);
}
/**
* Odata dedicated Services
*/
@Transactional(readOnly = true)
@Cacheable (value = "product_count", key = "{#criteria, #uuid}")
public int countProducts (DetachedCriteria criteria, String uuid)
{
if (criteria == null)
{
criteria = DetachedCriteria.forClass (Product.class);
}
// count only processed products
criteria.add (Restrictions.eq ("processed", true));
if (uuid != null)
{
List<Long> product_ids = collectionService.getProductIds (uuid);
criteria.add (Restrictions.in ("id", product_ids));
}
criteria.setProjection (Projections.rowCount ());
return productDao.count (criteria);
}
@Transactional (readOnly=true, propagation=Propagation.REQUIRED)
@Cacheable (value = "product_count", key = "{#filter, #collection?.getUUID ()}")
public int count (Collection collection, String filter)
{
if (collection == null)
{
return this.count (filter);
}
return this.count(filter, collection.getUUID());
}
/**
* Returns all products not contained in a collection.
* @return a set of products.
*/
@Transactional(readOnly = true)
public Set<Product> getNoCollectionProducts ()
{
DetachedCriteria criteria = DetachedCriteria.forClass (Product.class);
Iterator<Product> it =
collectionService.getAllProductInCollection ().iterator ();
HashSet<Long> cpid = new HashSet<> ();
while (it.hasNext ())
{
cpid.add (it.next ().getId ());
}
criteria.add (Restrictions.not (Restrictions.in ("id", cpid)));
criteria.add (Restrictions.eq ("processed", true));
return new HashSet<> (productDao.listCriteria (criteria, 0, -1));
}
/**
* Retrieve products ingested at a given date in the given collection.
*
* @param date ingestion date.
* @param collection the collection where found products.
* @return a set of researched products.
*/
@Transactional(readOnly = true)
public Set<Product> getProductByIngestionDate (Date date,
Collection collection)
{
Iterator<Product> it;
Set<Product> productSet = new HashSet<> ();
if (collection == null)
{
it = getNoCollectionProducts ().iterator ();
}
else
{
it = collectionService.systemGetCollection (
collection.getUUID ()).getProducts ().iterator ();
}
while (it.hasNext ())
{
Product p = it.next ();
if (p != null)
{
if (date.getTime () - p.getIngestionDate ().getTime () < DAY_MILLI)
{
productSet.add (p);
}
}
}
return productSet;
}
@Transactional(readOnly = true)
public Product getProductIdentifier (String identifier)
{
return productDao.getProductByIdentifier (identifier);
}
@Transactional
public void update (Product product)
{
productDao.update (product);
}
/*
* Reported from DefaultDataStrore
*/
private class ProcessingCallable extends FairCallable
{
Product product;
User owner;
Scanner scanner;
List<Collection>collections;
FileScannerWrapper wrapper;
public ProcessingCallable(Product product, User owner,
List<Collection> collections, Scanner scanner,
FileScannerWrapper wrapper)
{
super (scanner == null ? null : scanner.toString ());
this.product = product;
this.owner = owner;
this.collections = collections;
this.scanner = scanner;
this.wrapper = wrapper;
}
@IncrementCache (name = "product_count", key = "all", value = 1)
public Object call() throws Exception
{
ApplicationContextProvider.getBean (Ingester.class).ingest (
product, owner, collections, scanner, wrapper);
return null;
}
}
// Check if product present is the DB is still present into the repository.
@Transactional (readOnly=false, propagation=Propagation.REQUIRED)
public void checkDBProducts ()
{
LOGGER.info ("Syncing database with repositories...");
Iterator<Product> products = productDao.getAllProducts ();
while (products.hasNext ())
{
Product product = products.next ();
if ( !ProductService.checkUrl (product.getPath()))
{
LOGGER.info ("Removing Product " + product.getPath () +
" not found in repository.");
products.remove ();
}
else
LOGGER.info ("Product " + product.getPath () +
" found in repository.");
}
}
private static boolean checkUrl (URL url)
{
Objects.requireNonNull (url, "`url` parameter must not be null");
// OData Synchronized product, DELME
if (url.getPath ().endsWith ("$value"))
{
// Ignoring ...
return true;
}
// Case of simple file
try
{
File f = new File (url.toString ());
if (f.exists ()) return true;
}
catch (Exception e)
{
LOGGER.debug ("url \"" + url + "\" not formatted as a file");
}
// Case of local URL
try
{
URI local = new File (".").toURI ();
URI uri = local.resolve (url.toURI ());
File f = new File (uri);
if (f.exists ()) return true;
}
catch (Exception e)
{
LOGGER.debug ("url \"" + url + "\" not a local URL");
}
// Case of remote URL
try
{
URLConnection con = url.openConnection ();
con.connect ();
InputStream is = con.getInputStream ();
is.close ();
return true;
}
catch (Exception e)
{
LOGGER.debug ("url \"" + url + "\" not a remote URL");
}
// Unrecovrable case
return false;
}
/**
* Performs directory structure scan to retrieve relevant products, and run
* declared processing.
*
* @param archive the archive to be scan.
* @param productDao Data access object to products.
* @param indexDao Data access object to index in the products.
* @throws InterruptedException if user
*/
@Transactional (readOnly=false, propagation=Propagation.REQUIRED)
public int processArchiveSync ()
throws DataStoreLocalArchiveNotExistingException, InterruptedException
{
String archivePath = cfgManager.getArchiveConfiguration ().getPath ();
final File archive = new File(archivePath);
if (!archive.exists ())
{
throw new DataStoreLocalArchiveNotExistingException (
"Local archive \"" + archivePath + "\" does not exist.");
}
LOGGER.info ("Looking for new product in archive \"" +
archivePath + "\".");
final List<DrbCortexItemClass> supported =
scannerFactory.getScannerSupport ();
Scanner scanner =
scannerFactory.getScanner (archivePath);
scanner.setSupportedClasses (supported);
AsynchronousLinkedList<URLExt> list = scanner.getScanList ();
final Scanner s = scanner;
list.addListener (new Listener<URLExt> ()
{
@Override
public void addedElement (final Event<URLExt> e)
{
try
{
URL url = e.getElement ().getUrl ();
if (getProductByOrigin (url.toString ()) != null ||
productDao.exists (url))
{
throw new DataStoreAlreadyExistException (
"Already in database");
}
LOGGER.info ("Adding product \"" + url + "\".");
User owner = userDao.getRootUser ();
Product p = addProduct (url, owner, null);
processProduct (p, owner, null, s, null);
}
catch (DataStoreAlreadyExistException excp)
{
LOGGER.info ("Product already in database : \"" +
e.getElement ().getUrl ().toString () + "\".");
}
catch (DataStoreException excp)
{
LOGGER.error ("Cannot add product \"" +
e.getElement ().toString () + "\"", excp);
}
}
@Override
public void removedElement (Event<URLExt> e)
{
}
});
return scanner.scan ();
}
/**
* Remove unprocessed products
*/
@Transactional(readOnly = false, propagation = Propagation.REQUIRED)
@CacheEvict (
value = { "product_count", "product", "products" },
allEntries = true )
public void removeUnprocessed()
{
long start = System.currentTimeMillis();
Iterator<Product> products = getUnprocessedProducts();
while (products.hasNext())
{
products.next();
products.remove();
}
LOGGER.debug("Cleanup incomplete processed products in " + (System.currentTimeMillis() - start) + "ms");
}
@Transactional (readOnly = true)
public Iterator<Product> getUnprocessedProducts ()
{
return productDao.getUnprocessedProducts ();
}
@Transactional(readOnly = true)
public Product getProductByOrigin (final String origin)
{
DetachedCriteria criteria = DetachedCriteria.forClass (Product.class);
criteria.add (Restrictions.eq ("origin", origin));
return productDao.uniqueResult (criteria);
}
}