/*
* Data Hub Service (DHuS) - For Space data distribution.
* Copyright (C) 2016 GAEL Systems
*
* This file is part of DHuS software sources.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.gael.dhus.olingo.v1.entity;
import static fr.gael.dhus.olingo.v1.entityset.IngestEntitySet.FILENAME;
import static fr.gael.dhus.olingo.v1.entityset.IngestEntitySet.ID;
import static fr.gael.dhus.olingo.v1.entityset.IngestEntitySet.MD5;
import static fr.gael.dhus.olingo.v1.entityset.IngestEntitySet.STATUS;
import static fr.gael.dhus.olingo.v1.entityset.IngestEntitySet.STATUS_DATE;
import static fr.gael.dhus.olingo.v1.entityset.IngestEntitySet.STATUS_MESSAGE;
import static fr.gael.dhus.olingo.v1.entityset.IngestEntitySet.TARGET_COLLECTIONS;
import fr.gael.dhus.database.object.*;
import fr.gael.dhus.datastore.IncomingManager;
import fr.gael.dhus.olingo.v1.ExpectedException.IncompleteDocException;
import fr.gael.dhus.olingo.v1.ExpectedException.InvalidKeyException;
import fr.gael.dhus.olingo.v1.ExpectedException.InvalidTargetException;
import fr.gael.dhus.olingo.v1.Navigator;
import fr.gael.dhus.olingo.v1.Model;
import fr.gael.dhus.olingo.v1.map.FunctionalMap;
import fr.gael.dhus.olingo.v1.visitor.IngestFunctionalVisitor;
import fr.gael.dhus.service.CollectionService;
import fr.gael.dhus.service.ProductService;
import fr.gael.dhus.service.SecurityService;
import fr.gael.dhus.spring.context.ApplicationContextProvider;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.concurrent.atomic.AtomicLong;
import javax.xml.bind.DatatypeConverter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.olingo.odata2.api.edm.Edm;
import org.apache.olingo.odata2.api.edm.EdmEntitySet;
import org.apache.olingo.odata2.api.ep.entry.ODataEntry;
import org.apache.olingo.odata2.api.exception.ODataException;
import org.apache.olingo.odata2.api.rt.RuntimeDelegate;
import org.apache.olingo.odata2.api.uri.KeyPredicate;
import org.apache.olingo.odata2.api.uri.NavigationSegment;
import org.apache.olingo.odata2.api.uri.PathSegment;
import org.apache.olingo.odata2.api.uri.UriInfo;
import org.apache.olingo.odata2.api.uri.UriParser;
/**
* Ingest entity to upload/ingest new products.
*/
public class Ingest extends AbstractEntity
{
/** Log. */
private static final Logger LOGGER = LogManager.getLogger(Ingest.class);
/** Provides access to current user. */
private static final SecurityService SECURITY_SERVICE =
ApplicationContextProvider.getBean(SecurityService.class);
/** Provides the temp dir. */
private static final IncomingManager INCOMING_MANAGER =
ApplicationContextProvider.getBean(IncomingManager.class);
/** Provides the addProduct() method to add a product in the ingestion pipeline. */
private static final ProductService PRODUCT_SERVICE =
ApplicationContextProvider.getBean(ProductService.class);
/** To get db collections from their IDs. */
private static final CollectionService COLLECTION_SERVICE =
ApplicationContextProvider.getBean(CollectionService.class);
/** To auto-increment `id`, not the size of `UPLOAD` because delete is implemented. */
private static final AtomicLong CURSOR = new AtomicLong(0L);
/** Map of uploaded products, key is the Id. */
private static final Map<Long, Ingest> UPLOADS =
Collections.synchronizedMap(new HashMap<Long, Ingest>());
/** Key. */
private final Long id;
/** MD5sum of the data received by the constructor. */
private final String md5;
/** TargetCollection for the new product. */
private final Map<String, Collection> targetCollections = new HashMap<>();;
/** User who uploaded this data. */
private final fr.gael.dhus.database.object.User uploader;
/** When this Ingest entered the current status. */
private final Date statusDate = new Date();
/** Filename, required to start ingestion of data. */
private String filename;
/** Current status. */
private Status status;
/** Message bound to the current status. */
private String statusMessage;
/** Path to temp file holding the data to be ingested. */
private Path temp_file;
/**
* Creates a new Ingest with data to ingest.
* Will read the stream, store it in a temp file.
* @param in data to ingest.
* @throws ODataException an error occured.
*/
public Ingest(InputStream in) throws ODataException
{
uploader = SECURITY_SERVICE.getCurrentUser();
// write file to temp file
try
{
temp_file = Files.createTempFile(INCOMING_MANAGER.getTempDir().toPath(), null, ".ingest_data");
LOGGER.info(String.format("User %s uploading data to %s",
uploader.getUsername(), temp_file.toString()));
try (OutputStream os = Files.newOutputStream(temp_file))
{
// Computes the MD5 hash of the uploaded file as it is written to disk
MessageDigest md = MessageDigest.getInstance("MD5");
DigestOutputStream md5_os = new DigestOutputStream(os, md);
BufferedOutputStream bos = new BufferedOutputStream(md5_os);
int byt3;
while ((byt3 = in.read()) != -1)
{
bos.write(byt3);
}
bos.flush();
this.md5 = DatatypeConverter.printHexBinary(md5_os.getMessageDigest().digest());
}
}
catch (IOException | NoSuchAlgorithmException e)
{
LOGGER.fatal(e);
throw new ODataException("A system error occured", e);
}
id = CURSOR.getAndIncrement();
if (UPLOADS.put(id, this) != null)
{
LOGGER.fatal("Race condition!");
}
setStatus(Status.WAITING_FOR_METADATA);
statusMessage = "Set the Filename property to insert the product in the ingestion pipeline";
}
@Override
public void updateFromEntry(ODataEntry entry) throws ODataException
{
Map<String, Object> props = entry.getProperties();
String fname = (String)props.remove(FILENAME);
if (this.filename == null && (fname == null || fname.isEmpty()))
{
throw new IncompleteDocException("Property filename required");
}
this.filename = fname;
for (Map.Entry<String, Object> unkn: props.entrySet())
{
switch (unkn.getKey())
{
case ID:
case STATUS:
case STATUS_DATE:
case STATUS_MESSAGE:
LOGGER.warn("Property " + unkn.getKey() + " is read-only");
break;
default:
LOGGER.warn("Unknown property " + unkn.getKey());
}
}
List<String> target_collections = entry.getMetadata().getAssociationUris(TARGET_COLLECTIONS);
if (target_collections.size() > 0)
{
Edm edm = RuntimeDelegate.createEdm(new Model());
UriParser urip = RuntimeDelegate.getUriParser(edm);
for (String target_collection: target_collections)
{
List<PathSegment> path_segments = new ArrayList<>();
StringTokenizer st = new StringTokenizer(target_collection, "/");
while (st.hasMoreTokens ())
{
path_segments.add(UriParser.createPathSegment(st.nextToken(), null));
}
UriInfo uinfo = urip.parse(path_segments, Collections.EMPTY_MAP);
EdmEntitySet sync_ees = uinfo.getStartEntitySet();
KeyPredicate kp = uinfo.getKeyPredicates().get(0);
List<NavigationSegment> ns_l = uinfo.getNavigationSegments();
Collection coll = Navigator.<Collection>navigate(sync_ees, kp, ns_l, Collection.class);
if (coll == null)
{
throw new ODataException("Target collection not found: " + target_collection);
}
targetCollections.put(coll.getName(), coll);
}
}
// Ingesting ...
LOGGER.info(String.format("Ingesting product %s (IngestId=%d md5=%s) uploaded by %s",
filename, id, md5, uploader.getUsername()));
List<fr.gael.dhus.database.object.Collection> collections =
new ArrayList<>(targetCollections.size());
for (Collection c: targetCollections.values())
{
collections.add(COLLECTION_SERVICE.getCollection(c.getUUID()));
}
Path pname = temp_file.resolveSibling(filename);
try
{
Files.move(temp_file, pname);
temp_file = pname;
URL purl = pname.toUri().toURL();
fr.gael.dhus.database.object.Product p = PRODUCT_SERVICE.addProduct (
purl, uploader, purl.toString ());
PRODUCT_SERVICE.processProduct (p, uploader, collections, null, null);
}
catch (IOException ex)
{
LOGGER.error("Cannot ingest product", ex);
status = Status.ERROR;
statusMessage = ex.getMessage();
throw new ODataException("Cannot ingest product", ex);
}
setStatus(Status.INGESTED);
}
@Override
public Map<String, Object> toEntityResponse(String root_url)
{
HashMap<String, Object> res = new HashMap<>();
res.put(ID, id);
res.put(STATUS, status.toString());
res.put(STATUS_MESSAGE, statusMessage);
res.put(STATUS_DATE, statusDate);
res.put(MD5, md5);
res.put(FILENAME, filename);
return res;
}
@Override
public Object getProperty(String prop_name) throws ODataException
{
Object res;
switch (prop_name)
{
case ID:
res = id; break;
case STATUS:
res = status.toString(); break;
case STATUS_MESSAGE:
res = statusMessage; break;
case STATUS_DATE:
res = statusDate; break;
case MD5:
res = md5; break;
case FILENAME:
res = filename; break;
default:
LOGGER.warn("Requested property " + prop_name + " does not exist");
res = null;
}
return res;
}
@Override
public Object navigate(NavigationSegment ns) throws ODataException
{
Object res;
EdmEntitySet es = ns.getEntitySet();
if (es.getName().equals(Model.USER.getName()))
{
res = new User(uploader); // one to one
}
else if (es.getName().equals(Model.COLLECTION.getName()))
{
if (ns.getKeyPredicates().isEmpty())
{
res = Collections.unmodifiableMap(targetCollections);
}
else
{
KeyPredicate kp = ns.getKeyPredicates().get(0);
res = this.targetCollections.get(kp.getLiteral());
if (res == null)
{
throw new InvalidKeyException(kp.getLiteral(), ns.getEntitySet().getName());
}
}
}
else
{
throw new InvalidTargetException(this.getClass().getSimpleName(), ns.getEntitySet().getName());
}
return res;
}
/**
* Returns the requested Ingest, or {@code null} if no Ingest has such id.
* @param id unique identifier (key).
* @return an instance of Ingest or {@code null}.
*/
public static Ingest get(long id)
{
return UPLOADS.get(id);
}
/**
* Delete an instance of Ingest whose id is `id`.
* @param id of the Ingest instance to delete.
* @throws ODataException no Ingest was found for the given id.
*/
public static void delete(long id) throws ODataException
{
Ingest ingest;
if ((ingest = UPLOADS.remove(id)) == null)
{
throw new InvalidKeyException(String.valueOf(id), Ingest.class.getSimpleName());
}
else
{
try
{
Files.delete(ingest.temp_file);
}
catch (IOException ex)
{
LOGGER.error("Cannot delete ingest temp file " + ingest.temp_file, ex);
}
}
}
/**
* Returns an unmodifiable map on Ingests.
* The returned map is a shallow copy of the working map, thus it should not throw a
* ConcurrentModificationException when iterating over it.
* <p>The returned map implements SubMap.
* @return Ingests.
*/
public static Map<Long, Ingest> getMappable()
{
Map<Long, Ingest> res = new HashMap<>();
synchronized(UPLOADS)
{
res.putAll(UPLOADS);
}
return new FunctionalMap<>(res, new IngestFunctionalVisitor());
}
/**
* Navigate to linked entity User (navlink=`uploader`).
* @return linked entity User.
*/
public User navigateUploader()
{
return new User(this.uploader);
}
/**
* Unique identifier, Key.
* @return Id.
*/
public Long getId()
{
return id;
}
/**
* Filename, defaults to {@code null}. Must be set before the product is sent to ingestion.
* @return Filename.
*/
public String getFilename()
{
return filename;
}
/**
* MD5, as specified in the supplied OData entity at creation.
* @return MD5 Hash as string.
*/
public String getMd5()
{
return md5;
}
/**
* Status tells you if the uploaded data is being ingested or is already ingested.
* @return Status enum entry.
*/
public Status getStatus()
{
return status;
}
/**
* Since when the current status has been active.
* @return Status Date.
*/
public Date getStatusDate()
{
return statusDate;
}
/**
* Message associated with the current status, useful when the current status is ERROR.
* @return Status message.
*/
public String getStatusMessage()
{
return statusMessage;
}
/** Statuses. */
public static enum Status
{
/** Product has been upload, waiting for the user to set properties. */
WAITING_FOR_METADATA,
/** An error occured. */
ERROR,
/** Product has passed through the ingestion process, was it successful? nobody knows. */
INGESTED
}
private void setStatus(Status status)
{
this.status = status;
this.statusDate.setTime(System.currentTimeMillis());
}
}