/**
* MetadataTasks.java
* Author: Francesco Rosso (rosso@eurix.it)
*
* This file is part of PrestoPRIME Preservation Platform (P4).
*
* Copyright (C) 2012 EURIX Srl, Torino, Italy
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package eu.prestoprime.plugin.p4;
import it.eurix.archtools.data.DataException;
import it.eurix.archtools.data.model.DIP;
import it.eurix.archtools.data.model.DIP.DCField;
import it.eurix.archtools.data.model.IPException;
import it.eurix.archtools.data.model.SIP;
import it.eurix.archtools.tool.ToolException;
import it.eurix.archtools.tool.ToolOutput;
import it.eurix.archtools.workflow.exceptions.TaskExecutionFailedException;
import it.eurix.archtools.workflow.plugin.WfPlugin;
import it.eurix.archtools.workflow.plugin.WfPlugin.WfService;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.prestoprime.datamanagement.P4DataManager;
import eu.prestoprime.model.dnx.Dnx;
import eu.prestoprime.model.dnx.Key;
import eu.prestoprime.model.dnx.Record;
import eu.prestoprime.model.dnx.Section;
import eu.prestoprime.plugin.p4.tools.D10SumChecker;
import eu.prestoprime.plugin.p4.tools.FFprobe;
import eu.prestoprime.plugin.p4.tools.MXFTechMDExtractor;
import eu.prestoprime.search.P4Indexer;
@WfPlugin(name = "P4Plugin")
public class MetadataTasks {
public static final Logger logger = LoggerFactory.getLogger(MetadataTasks.class);
@WfService(name = "ffprobe", version = "2.1.0")
public void ffprobe(Map<String, String> sParams, Map<String, String> dParamsString, Map<String, File> dParamFile) throws TaskExecutionFailedException {
// get sipID
String sipID = dParamsString.get("sipID");
SIP sip = null;
try {
// get sip
sip = P4DataManager.getInstance().getSIPByID(sipID);
// get MQ file
String videoFile = null;
String[] MQformats = sParams.get("MQformats").split(",");
logger.debug("Checking for MQ formats: " + Arrays.toString(MQformats));
for (String format : MQformats) {
List<String> videoFileList = sip.getAVMaterial(format, "FILE");
if (videoFileList.size() > 0) {
videoFile = videoFileList.get(0);
break;
}
}
if (videoFile == null) {
throw new TaskExecutionFailedException("Unable to find supported MQ format...");
}
// run FFprobe
FFprobe ffprobe = new FFprobe();
ToolOutput<FFprobe.AttributeType> output = ffprobe.extract(videoFile);
String ffprobeOutput = output.getAttribute(FFprobe.AttributeType.json);
Section section = new Section();
section.setId("ffprobe");
// get JSON Object
JSONObject jsonOutput = new JSONObject(ffprobeOutput);
// FFprobe show format
JSONObject jsonFormat = jsonOutput.getJSONObject("format");
Iterator<String> itFormat = jsonFormat.keys();
while (itFormat.hasNext()) {
String key = itFormat.next();
String value = jsonFormat.getString(key);
Record record = new Record();
Key keyType = new Key();
keyType.setId("significantPropertiesType");
keyType.setValue("ffprobe:" + key);
record.getKey().add(keyType);
Key keyValue = new Key();
keyValue.setId("significantPropertiesValue");
keyValue.setValue(value);
record.getKey().add(keyValue);
section.getRecord().add(record);
}
// set format
String formatN = jsonFormat.getString("format_name");
String formatLN = jsonFormat.getString("format_long_name");
// size
String size = jsonFormat.getString("size");
dParamsString.put("isMXF", "false");
if (formatN != null) {
synchronized (sip) {
List<String> formats = sip.getDCField(DCField.format);
formats.add(formatLN + "(" + formatN.toUpperCase() + ")");
sip.setDCField(DCField.format, formats);
}
if (formatN.equalsIgnoreCase("MXF")) {
dParamsString.put("isMXF", "true");
}
}
// FFprobe show streams
JSONArray jsonStreams = jsonOutput.getJSONArray("streams");
String secDuration = null;
String frameRate = null;
int numOfStreams = jsonStreams.length();
for (int i = 0; i < numOfStreams; i++) {
JSONObject jsonStream = jsonStreams.getJSONObject(i);
String index = jsonStream.getString("index");
Iterator<String> itStream = jsonStream.keys();
while (itStream.hasNext()) {
String key = itStream.next();
String value = jsonStream.getString(key);
Record record = new Record();
Key keyType = new Key();
keyType.setId("significantPropertiesType");
keyType.setValue("ffprobe:" + key + "_" + index);
record.getKey().add(keyType);
Key keyValue = new Key();
keyValue.setId("significantPropertiesValue");
keyValue.setValue(value);
record.getKey().add(keyValue);
section.getRecord().add(record);
}
if (jsonStream.getString("codec_type").equals("video")) {
secDuration = jsonStream.getString("duration");
frameRate = jsonStream.getString("r_frame_rate");
}
}
try {
dParamsString.put("duration", "" + (int) Double.parseDouble(secDuration));
} catch (NumberFormatException e) {
dParamsString.put("duration", "0");
}
if (frameRate != null) {
String fpsString = frameRate.split("/")[0];
try {
dParamsString.put("fps", "" + Integer.parseInt(fpsString));
} catch (NumberFormatException e) {
dParamsString.put("fps", "25");
}
}
// set new DNX section
Dnx dnx = new Dnx();
dnx.getSection().add(section);
sip.addDNX(dnx, "ffprobe", true);
// add DNX section with video information
Record videoRecord = new Record();
Key key1 = new Key();
key1.setId("duration");
key1.setValue(secDuration);
videoRecord.getKey().add(key1);
Key key2 = new Key();
key2.setId("nb_streams");
key2.setValue(String.valueOf(numOfStreams));
videoRecord.getKey().add(key2);
Key key3 = new Key();
key3.setId("format_long_name");
key3.setValue(formatLN);
videoRecord.getKey().add(key3);
Key key4 = new Key();
key4.setId("size");
key4.setValue(size);
videoRecord.getKey().add(key4);
Section videoSec = new Section();
videoSec.setId("videoMD");
videoSec.getRecord().add(videoRecord);
Dnx videoDnx = new Dnx();
videoDnx.getSection().add(videoSec);
sip.addDNX(videoDnx, "videoMD", false);
} catch (JSONException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to parse JSON output from ffprobe...");
} catch (DataException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to retrieve the SIP...");
} catch (IPException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to work with SIP...");
} catch (ToolException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to run FFProbe...");
} finally {
// release SIP
P4DataManager.getInstance().releaseIP(sip);
}
}
@WfService(name = "mxf_techmd_extract", version = "2.0.0")
public void mxfTechMD(Map<String, String> sParams, Map<String, String> dParamsString, Map<String, File> dParamFile) throws TaskExecutionFailedException {
if (Boolean.parseBoolean(dParamsString.get("isMXF"))) {
// get sipID
String sipID = dParamsString.get("sipID");
SIP sip = null;
try {
// get SIP
sip = P4DataManager.getInstance().getSIPByID(sipID);
// get MQ file
String mxfFile = sip.getAVMaterial("application/mxf", "FILE").get(0);
// run MXFTechMD
MXFTechMDExtractor mxfTechMD = new MXFTechMDExtractor();
logger.debug("Inside extractMxfTechMd...");
logger.debug("File passed to MXFTechMD: " + mxfFile);
mxfTechMD.extract(mxfFile);
logger.info("Extracted MXF Technical Metadata...");
List<String> attributeList = mxfTechMD.getSupportedAttributeNames();
Section section = new Section();
section.setId("significantProperties");
Record record = null;
for (String attributeName : attributeList) {
record = new Record();
Key keyType = new Key();
keyType.setId("significantPropertiesType");
keyType.setValue("MXFTechMDExtractor:" + attributeName);
record.getKey().add(keyType);
Key keyValue = new Key();
keyValue.setId("significantPropertiesValue");
keyValue.setValue(mxfTechMD.getAttributeByName(attributeName));
record.getKey().add(keyValue);
section.getRecord().add(record);
}
section.setId("mxf-techmd-extractor");
Dnx dnx = new Dnx();
dnx.getSection().add(section);
logger.info("Added new section to dnx...");
// set format
String containers = mxfTechMD.getAttributeByName("EssenceContainers");
dParamsString.put("isD10", "false");
if (containers != null) {
synchronized (sip) {
List<String> formats = sip.getDCField(DCField.format);
formats.add(containers);
sip.setDCField(DCField.format, formats);
}
if (containers.contains("D10 Mapping")) {
dParamsString.put("isD10", "true");
}
}
// add new DNX section
sip.addDNX(dnx, "mxftechmd", true);
} catch (DataException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to find SIP...");
} catch (IPException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to work with SIP...");
} catch (ToolException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to run MXFTechMD extractor...");
} finally {
// release SIP
P4DataManager.getInstance().releaseIP(sip);
}
}
}
@WfService(name = "d10_fixity_checker", version = "1.3.0")
public void d10FixityChecker(Map<String, String> arg0, Map<String, String> dParamsString, Map<String, File> arg2) throws TaskExecutionFailedException {
if (Boolean.parseBoolean(dParamsString.get("isD10"))) {
String sipID = dParamsString.get("sipID");
SIP sip = null;
try {
sip = P4DataManager.getInstance().getSIPByID(sipID);
String videoFilePath = sip.getAVMaterial("application/mxf", "FILE").get(0);
D10SumChecker d10SumChecker = new D10SumChecker();
ToolOutput<D10SumChecker.AttributeType> output = d10SumChecker.extract(videoFilePath);
Record d10Record = new Record();
Key d10agKey = new Key();
d10agKey.setId("agent");
d10agKey.setValue("D10SumChecker");
d10Record.getKey().add(d10agKey);
Key d10ftKey = new Key();
d10ftKey.setId("fixityType");
d10ftKey.setValue("D10SumChecker EditUnit MD5");
d10Record.getKey().add(d10ftKey);
Key d10eunKey = new Key();
d10eunKey.setId("fixityValue");
d10eunKey.setValue(output.getAttribute(D10SumChecker.AttributeType.D10SumChecker));
d10Record.getKey().add(d10eunKey);
Section section = new Section();
section.setId("fileFixity");
section.getRecord().add(d10Record);
Dnx dnx = new Dnx();
dnx.getSection().add(section);
sip.addDNX(dnx, "d10sumchecker", true);
} catch (DataException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to get the SIP...");
} catch (IPException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to get MQ file...");
} catch (ToolException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to run D10SumChecker...");
} finally {
P4DataManager.getInstance().releaseIP(sip);
}
}
}
@WfService(name = "metadata_indexing", version = "2.1.0")
public void indexMetadata(Map<String, String> sParams, Map<String, String> dParamsString, Map<String, File> dParamsFile) throws TaskExecutionFailedException {
String sipID = dParamsString.get("sipID");
P4Indexer indexer = new P4Indexer();
boolean success = false;
DIP sip = null;
if (sipID != null) {
try {
logger.debug("Indexing metadata of SIP " + sipID);
logger.debug("Retrieving SIP " + sipID + " from Persistence DB");
sip = P4DataManager.getInstance().getDIPByID(sipID);
logger.debug("Got DIP object " + sip);
// TODO use indexer.indexIP as soon as dc record getter works on
// P4InformationPackage
success = indexer.indexIP(sip);
} catch (DataException e) {
logger.warn("DC Records could not be obtained.");
logger.warn(e.getMessage());
}
if (!success) {
throw new TaskExecutionFailedException("Unable to index Metadata");
}
} else {
throw new TaskExecutionFailedException("SIP ID was not set in workflow");
}
}
@WfService(name = "rebuild_search_index", version = "2.2.0")
public void reindexMetadata(Map<String, String> sParams, Map<String, String> dParamsString, Map<String, File> dParamsFile) throws TaskExecutionFailedException {
try {
boolean success = true;
List<String> failList = new ArrayList<>();
P4Indexer indexer = new P4Indexer();
success &= indexer.clearIndex();
logger.debug("Successfully cleared search index...");
List<String> aipIdList = P4DataManager.getInstance().getAllAIP(null);
if (aipIdList.isEmpty()) {
logger.debug("No AIPs found.");
}
for (String aipId : aipIdList) {
logger.debug("Indexing AIP " + aipId);
DIP dip = P4DataManager.getInstance().getDIPByID(aipId);
boolean aipSuccess = indexer.indexIP(dip);
success &= aipSuccess;
if (!aipSuccess) {
failList.add(aipId);
}
}
if (success) {
logger.debug("Indexing done! Number of indexed AIPs: " + aipIdList.size());
} else {
StringBuilder sb = new StringBuilder();
String delim = "";
for (String id : failList) {
sb.append(delim);
sb.append(id);
delim = ", ";
}
logger.error("Indexing failed for AIPs: " + sb.toString());
throw new TaskExecutionFailedException("Unable to rebuild search index...");
}
} catch (DataException e) {
e.printStackTrace();
throw new TaskExecutionFailedException("Unable to retrieve AIP list from database...");
}
}
}