/*
* Copyright (c) 2013 Data Harmonisation Panel
*
* All rights reserved. This program and the accompanying materials are made
* available under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 3 of the License,
* or (at your option) any later version.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this distribution. If not, see <http://www.gnu.org/licenses/>.
*
* Contributors:
* Data Harmonisation Panel <http://www.dhpanel.eu>
*/
package eu.esdihumboldt.hale.common.align.io.impl.internal;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import com.google.common.base.Strings;
import eu.esdihumboldt.hale.common.align.extension.function.custom.CustomPropertyFunction;
import eu.esdihumboldt.hale.common.align.model.Alignment;
import eu.esdihumboldt.hale.common.align.model.AlignmentUtil;
import eu.esdihumboldt.hale.common.align.model.BaseAlignmentCell;
import eu.esdihumboldt.hale.common.align.model.Cell;
import eu.esdihumboldt.hale.common.align.model.ModifiableCell;
import eu.esdihumboldt.hale.common.align.model.MutableAlignment;
import eu.esdihumboldt.hale.common.align.model.MutableCell;
import eu.esdihumboldt.hale.common.align.model.TransformationMode;
import eu.esdihumboldt.hale.common.align.model.impl.DefaultAlignment;
import eu.esdihumboldt.hale.common.core.io.PathUpdate;
import eu.esdihumboldt.hale.common.core.io.report.IOReporter;
import eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl;
import eu.esdihumboldt.hale.common.core.io.supplier.DefaultInputSupplier;
import eu.esdihumboldt.hale.common.schema.model.TypeIndex;
import eu.esdihumboldt.util.io.IOUtils;
/**
* Base class for converting alignment representations to alignments.
*
* @author Kai Schwierczek
* @param <A> the alignment representation type
* @param <C> the cell representation type
* @param <M> the cell modifier representation type
*/
public abstract class AbstractBaseAlignmentLoader<A, C, M> {
/**
* Load a alignment representation from the given stream. This method must
* close the stream after it is done.
*
* @param in the input stream
* @param reporter the I/O reporter to report any errors to, may be
* <code>null</code>
* @return a alignment representation
* @throws IOException if some kind of exception occurs while loading the
* alignment
*/
protected abstract A loadAlignment(InputStream in, IOReporter reporter) throws IOException;
/**
* Returns a map of prefix, URI pairs of base alignments for the given
* alignment. The returned map must be modifiable.
*
* @param alignment the alignment representation in question
* @return a map of prefix, URI pairs of base alignments
*/
protected abstract Map<String, URI> getBases(A alignment);
/**
* Returns a collection of cell representations of the given alignment
* representation.
*
* @param alignment the alignment representation in question
* @return cell representations
*/
protected abstract Collection<C> getCells(A alignment);
/**
* Returns a collection of property function definitions of the given
* alignment representation.
*
* @param alignment the alignment representation in question
* @param sourceTypes the source types to use for resolving definition
* references
* @param targetTypes the target types to use for resolving definition
* references
* @return list of property functions representations
*/
protected abstract Collection<CustomPropertyFunction> getPropertyFunctions(A alignment,
TypeIndex sourceTypes, TypeIndex targetTypes);
/**
* Returns the cell id of the given cell.
*
* @param cell the cell in question
* @return the cell id of the given cell
*/
protected abstract String getCellId(C cell);
/**
* Create a cell from the given cell representation
*
* @param cell the cell representation
* @param sourceTypes the source types to use for resolving definition
* references
* @param targetTypes the target types to use for resolving definition
* references
* @param reporter the I/O reporter to report any errors to, may be
* <code>null</code>
* @return a cell for the given cell representation
*/
protected abstract MutableCell createCell(C cell, TypeIndex sourceTypes, TypeIndex targetTypes,
IOReporter reporter);
/**
* Returns a collection of modifier representations of the given alignment
* representation.
*
* @param alignment the alignment representation in question
* @return modifier representations
*/
protected abstract Collection<M> getModifiers(A alignment);
/**
* Returns the raw cell id that is modified by the given modifier.
*
* @param modifier the modifier representation in question
* @return the cell id that is modified
*/
protected abstract String getModifiedCell(M modifier);
/**
* Returns the disabled for list of the given modifier representation.
*
* @param modifier the modifier representation in question
* @return the disabled for list
*/
protected abstract Collection<String> getDisabledForList(M modifier);
/**
* Get the transformation mode specified in a modifier.
*
* @param modifier the modifier
* @return the transformation mode or <code>null</code> if none is specified
*/
protected abstract TransformationMode getTransformationMode(M modifier);
/**
* Private class to save alignment information.
*/
private class AlignmentInfo {
AlignmentInfo(String prefix, URIPair uri) {
this.prefix = prefix;
this.uri = uri;
}
final String prefix;
final URIPair uri;
}
/**
* Private class for a pair of URIs. The used URI and the absolute URI.
*
* {@link #equals(Object)} and {@link #hashCode()} only use the absolute
* URI.
*/
private class URIPair {
URIPair(URI absoluteURI, URI usedURI) {
this.absoluteURI = absoluteURI;
this.usedURI = usedURI;
}
final URI absoluteURI;
final URI usedURI;
/**
* @see java.lang.Object#hashCode()
*/
@Override
public int hashCode() {
return 31 + ((absoluteURI == null) ? 0 : absoluteURI.hashCode());
}
@SuppressWarnings("unchecked")
@Override
public boolean equals(Object obj) {
if (obj instanceof AbstractBaseAlignmentLoader.URIPair)
return absoluteURI.equals(((URIPair) obj).absoluteURI);
else
return false;
}
}
/**
* Adds the given base alignment to the given alignment.
*
* @param alignment the alignment to add a base alignment to
* @param newBase URI of the new base alignment
* @param projectLocation the project location or <code>null</code>
* @param sourceTypes the source types to use for resolving definition
* references
* @param targetTypes the target types to use for resolving definition
* references
* @param reporter the I/O reporter to report any errors to, may be
* <code>null</code>
* @throws IOException if adding the base alignment fails
*/
protected final void internalAddBaseAlignment(MutableAlignment alignment, URI newBase,
URI projectLocation, TypeIndex sourceTypes, TypeIndex targetTypes, IOReporter reporter)
throws IOException {
Map<A, Map<String, String>> prefixMapping = new HashMap<A, Map<String, String>>();
Map<A, AlignmentInfo> alignmentToInfo = new HashMap<A, AlignmentInfo>();
generatePrefixMapping(newBase, projectLocation, alignment.getBaseAlignments(),
prefixMapping, alignmentToInfo, reporter);
processBaseAlignments(alignment, sourceTypes, targetTypes, prefixMapping, alignmentToInfo,
reporter);
}
/**
* Creates and adds cells and modifiers of the base alignments to the main
* alignment.
*
* @param alignment the alignment to add base alignments to
* @param sourceTypes the source types to use for resolving definition
* references
* @param targetTypes the target types to use for resolving definition
* references
* @param prefixMapping gets filled with a mapping from local to global
* prefixes
* @param alignmentToInfo gets filled with a mapping from base alignment
* representations to prefixes and URIs
* @param reporter the I/O reporter to report any errors to, may be
* <code>null</code>
* @throws IOException if one of the base alignments does not have cell ids
*/
private void processBaseAlignments(MutableAlignment alignment, TypeIndex sourceTypes,
TypeIndex targetTypes, Map<A, Map<String, String>> prefixMapping,
Map<A, AlignmentInfo> alignmentToInfo, IOReporter reporter) throws IOException {
for (Entry<A, AlignmentInfo> base : alignmentToInfo.entrySet()) {
Collection<C> baseCells = getCells(base.getKey());
boolean hasIds = true;
for (C baseCell : baseCells)
if (Strings.isNullOrEmpty(getCellId(baseCell))) {
hasIds = false;
break;
}
if (!hasIds) {
throw new IOException(
"At least one base alignment (" + base.getValue().uri.absoluteURI
+ ") has no cell ids. Please load and save it to generate them.");
}
}
for (Entry<A, AlignmentInfo> base : alignmentToInfo.entrySet()) {
if (alignment.getBaseAlignments().containsValue(base.getValue().uri.usedURI)) {
// base alignment already present
// can currently happen with base alignments included in base
// alignments
reporter.warn(new IOMessageImpl("Base alignment at " + base.getValue().uri.usedURI
+ " has already been added", null));
}
else {
Collection<CustomPropertyFunction> baseFunctions = getPropertyFunctions(
base.getKey(), sourceTypes, targetTypes);
Collection<C> baseCells = getCells(base.getKey());
Collection<BaseAlignmentCell> createdCells = new ArrayList<BaseAlignmentCell>(
baseCells.size());
for (C baseCell : baseCells) {
// add cells of base alignments
MutableCell cell = createCell(baseCell, sourceTypes, targetTypes, reporter);
if (cell != null) {
createdCells.add(new BaseAlignmentCell(cell, base.getValue().uri.usedURI,
base.getValue().prefix));
}
}
alignment.addBaseAlignment(base.getValue().prefix, base.getValue().uri.usedURI,
createdCells, baseFunctions);
}
}
// add modifiers of base alignments
for (Entry<A, AlignmentInfo> base : alignmentToInfo.entrySet())
applyModifiers(alignment, getModifiers(base.getKey()), prefixMapping.get(base.getKey()),
base.getValue().prefix, true, reporter);
}
/**
* Function to fill the prefixMapping and alignmentToInfo maps.
*
* @param addBase the URI of the new base alignment to add
* @param projectLocation the project location or <code>null</code>
* @param existingBases the map of existing bases
* @param prefixMapping gets filled with a mapping from local to global
* prefixes
* @param alignmentToInfo gets filled with a mapping from base alignment
* representations to prefixes and URIs
* @param reporter the reporter
* @return whether newBase actually is a new base and the add process should
* continue
*/
private boolean generatePrefixMapping(URI addBase, URI projectLocation,
Map<String, URI> existingBases, Map<A, Map<String, String>> prefixMapping,
Map<A, AlignmentInfo> alignmentToInfo, IOReporter reporter) {
// Project location may be null if the project wasn't saved yet
// Then, it still is okay, if all bases are absolute.
URI currentAbsolute = projectLocation;
URI usedAddBaseURI = addBase;
URI absoluteAddBaseURI = resolve(currentAbsolute, usedAddBaseURI);
// set of already seen URIs
Set<URI> knownURIs = new HashSet<URI>();
// reverse map of base
Map<URI, String> uriToPrefix = new HashMap<URI, String>();
// create URI to prefix map and known URIs
for (Entry<String, URI> baseEntry : existingBases.entrySet()) {
// make sure to use absolute URIs here for comparison
URI absoluteBase = resolve(currentAbsolute, baseEntry.getValue());
knownURIs.add(absoluteBase);
uriToPrefix.put(absoluteBase, baseEntry.getKey());
}
if (uriToPrefix.containsKey(absoluteAddBaseURI)) {
reporter.info(new IOMessageImpl(
"The base alignment (" + addBase + ") is already included.", null));
return false;
}
Set<String> existingPrefixes = new HashSet<String>(existingBases.keySet());
String newPrefix = generatePrefix(existingPrefixes);
existingPrefixes.add(newPrefix);
uriToPrefix.put(absoluteAddBaseURI, newPrefix);
/*
* XXX Adding a base alignment could only use a PathUpdate for the
* movement of the base alignment which is not known in the current
* project. Maybe could try the one of the current project either way?
*/
// find all alignments to load (also missing ones) and load the beans
LinkedList<URIPair> queue = new LinkedList<URIPair>();
queue.add(new URIPair(absoluteAddBaseURI, usedAddBaseURI));
while (!queue.isEmpty()) {
URIPair baseURI = queue.pollFirst();
A baseA;
try {
baseA = loadAlignment(new DefaultInputSupplier(baseURI.absoluteURI).getInput(),
reporter);
} catch (IOException e) {
reporter.error(new IOMessageImpl(
"Couldn't load an included base alignment (" + baseURI.absoluteURI + ").",
e));
reporter.setSuccess(false);
return false;
}
// add to alignment info map
alignmentToInfo.put(baseA,
new AlignmentInfo(uriToPrefix.get(baseURI.absoluteURI), baseURI));
prefixMapping.put(baseA, new HashMap<String, String>());
// load "missing" base alignments, too, add prefix mapping
for (Entry<String, URI> baseEntry : getBases(baseA).entrySet()) {
// rawURI may be relative
URI rawURI = baseEntry.getValue();
URI absoluteURI = baseURI.absoluteURI.resolve(rawURI);
URI usedURI = absoluteURI;
// If the added base alignment URI, the used URI for A
// and rawURI are relative, continue using a relative URI.
if (!usedAddBaseURI.isAbsolute() && !baseURI.usedURI.isAbsolute()
&& !rawURI.isAbsolute()) {
usedURI = IOUtils.getRelativePath(absoluteURI, currentAbsolute);
}
// check whether this base alignment is missing
if (!knownURIs.contains(absoluteURI)) {
reporter.info(new IOMessageImpl(
"A base alignment referenced another base alignment (" + absoluteURI
+ ") that was not yet known. It is now included, too.",
null));
queue.add(new URIPair(absoluteURI, usedURI));
knownURIs.add(absoluteURI);
String prefix = generatePrefix(existingPrefixes);
existingPrefixes.add(prefix);
uriToPrefix.put(absoluteURI, prefix);
}
// add prefix mapping
prefixMapping.get(baseA).put(baseEntry.getKey(), uriToPrefix.get(absoluteURI));
}
}
return true;
}
private URI resolve(URI base, URI relative) {
if (relative.isAbsolute())
return relative;
else
return base.resolve(relative);
}
/**
* Creates an alignment from the given alignment representation.
*
* @param start the main alignment representation
* @param sourceTypes the source types to use for resolving definition
* references
* @param targetTypes the target types to use for resolving definition
* references
* @param updater the path updater to use for base alignments
* @param reporter the I/O reporter to report any errors to, may be
* <code>null</code>
* @return the alignment for the given alignment representation
* @throws IOException if a base alignment couldn't be loaded
*/
protected final MutableAlignment createAlignment(A start, TypeIndex sourceTypes,
TypeIndex targetTypes, PathUpdate updater, IOReporter reporter) throws IOException {
Map<A, Map<String, String>> prefixMapping = new HashMap<A, Map<String, String>>();
Map<A, AlignmentInfo> alignmentToInfo = new HashMap<A, AlignmentInfo>();
// fill needed maps
generatePrefixMapping(start, prefixMapping, alignmentToInfo, updater, reporter);
// create alignment
DefaultAlignment alignment = new DefaultAlignment();
// add cells of base alignments
processBaseAlignments(alignment, sourceTypes, targetTypes, prefixMapping, alignmentToInfo,
reporter);
loadCustomFunctions(start, alignment, sourceTypes, targetTypes);
// add cells of main alignment
for (C mainCell : getCells(start)) {
MutableCell cell = createCell(mainCell, sourceTypes, targetTypes, reporter);
if (cell != null)
alignment.addCell(cell);
}
// add modifiers of main alignment
applyModifiers(alignment, getModifiers(start), prefixMapping.get(start), null, false,
reporter);
return alignment;
}
/**
* Load custom functions and add them to the alignment.
*
* @param source the alignment source
* @param alignment the alignment
* @param sourceTypes the source types
* @param targetTypes the target types
*/
protected void loadCustomFunctions(A source, DefaultAlignment alignment, TypeIndex sourceTypes,
TypeIndex targetTypes) {
Collection<CustomPropertyFunction> functions = getPropertyFunctions(source, sourceTypes,
targetTypes);
for (CustomPropertyFunction cf : functions) {
alignment.addCustomPropertyFunction(cf);
}
}
/**
* Function to fill the prefixMapping and alignmentToInfo maps.
*
* @param start the main alignment representation
* @param prefixMapping gets filled with a mapping from local to global
* prefixes
* @param alignmentToInfo gets filled with a mapping from base alignment
* representations to prefixes and URIs
* @param updater the location updater to use for base alignments
* @param reporter the reporter
* @throws IOException if a base alignment couldn't be loaded
*/
private void generatePrefixMapping(A start, Map<A, Map<String, String>> prefixMapping,
Map<A, AlignmentInfo> alignmentToInfo, PathUpdate updater, IOReporter reporter)
throws IOException {
// XXX What if the project file path would change?
// Alignment is a project file, so it is in the same directory.
URI currentAbsolute = updater.getNewLocation();
Map<String, URI> base = getBases(start);
// also a mapping for this alignment itself in case the same URI is
// defined for two prefixes
prefixMapping.put(start, new HashMap<String, String>());
// set of already seen URIs
Set<URI> knownURIs = new HashSet<URI>();
// reverse map of base
Map<URI, String> uriToPrefix = new HashMap<URI, String>();
// queue of base alignments to process
LinkedList<URIPair> queue = new LinkedList<URIPair>();
// check base for doubles, and invert it for later
for (Entry<String, URI> baseEntry : base.entrySet()) {
URI rawBaseURI = baseEntry.getValue();
URI usedBaseURI = updater.findLocation(rawBaseURI, true, false, true);
if (usedBaseURI == null) {
throw new IOException(
"Couldn't load an included alignment (" + rawBaseURI + "). File not found.",
null);
}
URI absoluteBaseURI = usedBaseURI;
if (!absoluteBaseURI.isAbsolute())
absoluteBaseURI = currentAbsolute.resolve(absoluteBaseURI);
if (knownURIs.contains(absoluteBaseURI)) {
reporter.warn(new IOMessageImpl(
"The same base alignment (" + rawBaseURI + ") was included twice.", null));
prefixMapping.get(start).put(baseEntry.getKey(), uriToPrefix.get(absoluteBaseURI));
}
else {
knownURIs.add(absoluteBaseURI);
prefixMapping.get(start).put(baseEntry.getKey(), baseEntry.getKey());
uriToPrefix.put(absoluteBaseURI, baseEntry.getKey());
queue.add(new URIPair(absoluteBaseURI, usedBaseURI));
}
}
// find all alignments to load (also missing ones) and load the beans
while (!queue.isEmpty()) {
URIPair baseURI = queue.pollFirst();
A baseA;
try {
baseA = loadAlignment(new DefaultInputSupplier(baseURI.absoluteURI).getInput(),
reporter);
} catch (IOException e) {
throw new IOException("Couldn't load an included alignment (" + baseURI + ").", e);
}
// add to alignment info map
alignmentToInfo.put(baseA,
new AlignmentInfo(uriToPrefix.get(baseURI.absoluteURI), baseURI));
prefixMapping.put(baseA, new HashMap<String, String>());
// load "missing" base alignments, too, add prefix mapping
for (Entry<String, URI> baseEntry : getBases(baseA).entrySet()) {
// rawURI may be relative
URI rawURI = baseEntry.getValue();
URI absoluteURI = baseURI.absoluteURI.resolve(rawURI);
// try updater again, it might help, and it shows whether the
// file is readable
absoluteURI = updater.findLocation(absoluteURI, true, false, false);
if (absoluteURI == null)
throw new IOException("Couldn't find an included alignment (" + rawURI + ").");
URI usedURI = absoluteURI;
// If the used URI for A and rawURI are relative, continue using
// a relative URI.
if (!baseURI.usedURI.isAbsolute() && !rawURI.isAbsolute())
usedURI = IOUtils.getRelativePath(absoluteURI, currentAbsolute);
if (!knownURIs.contains(absoluteURI)) {
reporter.info(new IOMessageImpl(
"A base alignment referenced another base alignment (" + absoluteURI
+ ") that was not yet known. It is now included, too.",
null));
queue.add(new URIPair(absoluteURI, usedURI));
knownURIs.add(absoluteURI);
String prefix = generatePrefix(base.keySet());
base.put(prefix, usedURI);
uriToPrefix.put(absoluteURI, prefix);
prefixMapping.get(start).put(prefix, prefix);
}
// add prefix mapping
prefixMapping.get(baseA).put(baseEntry.getKey(), uriToPrefix.get(absoluteURI));
}
}
}
/**
* Apply modifiers on the alignment.
*
* @param alignment the alignment to work on
* @param modifiers the modifiers to apply
* @param prefixMapping the mapping of prefixes (see
* {@link #getCell(Alignment, String, String, Map, IOReporter)})
* @param defaultPrefix the default prefix (may be <code>null</code>) (see
* {@link #getCell(Alignment, String, String, Map, IOReporter)})
* @param base whether the added modifiers are from a base alignment or the
* main alignment
* @param reporter the I/O reporter to report any errors to, may be
* <code>null</code>
*/
private void applyModifiers(Alignment alignment, Collection<M> modifiers,
Map<String, String> prefixMapping, String defaultPrefix, boolean base,
IOReporter reporter) {
for (M modifier : modifiers) {
Cell cell = getCell(alignment, getModifiedCell(modifier), defaultPrefix, prefixMapping,
reporter);
if (cell == null)
continue;
// disabledFor
for (String disabledForId : getDisabledForList(modifier)) {
Cell other = getCell(alignment, disabledForId, defaultPrefix, prefixMapping,
reporter);
if (other == null)
continue;
else if (!AlignmentUtil.isTypeCell(other)) {
reporter.warn(new IOMessageImpl(
"A cell referenced in disable-for is not a type cell.", null));
continue;
}
else if (!alignment.getPropertyCells(other, true, false).contains(cell)) {
reporter.warn(new IOMessageImpl(
"A cell referenced in disable-for does not contain the cell that gets modified.",
null));
continue;
}
// base is true -> modified cell has to be of a base alignment
// so it has to be a BaseAlignmentCell
if (base)
((BaseAlignmentCell) cell).setBaseDisabledFor(other, true);
else
((ModifiableCell) cell).setDisabledFor(other, true);
}
// transformation mode
TransformationMode mode = getTransformationMode(modifier);
if (mode != null) {
if (base)
((BaseAlignmentCell) cell).setBaseTransformationMode(mode);
else
((ModifiableCell) cell).setTransformationMode(mode);
}
// XXX handle additional properties
}
}
/**
* Returns the cell in question or null, if it could not be found in which
* case a suitable warning was generated.
*
* @param alignment the alignment which contains the cell
* @param cellId the cell id
* @param defaultPrefix the prefix to use if the cell id does not contain a
* prefix, may be <code>null</code>
* @param prefixMapping the prefix map to transform the prefix of the cell
* id with, if it has one
* @param reporter the I/O reporter to report any errors to, may be
* <code>null</code>
* @return the cell in question or <code>null</code>
*/
private Cell getCell(Alignment alignment, String cellId, String defaultPrefix,
Map<String, String> prefixMapping, IOReporter reporter) {
String prefix = defaultPrefix;
// check if the cell id references another base alignment
int prefixSplit = cellId.indexOf(':');
if (prefixSplit != -1) {
prefix = prefixMapping.get(cellId.substring(0, prefixSplit));
if (prefix == null) {
reporter.warn(new IOMessageImpl("A modifier used an unknown cell prefix", null));
return null;
}
cellId = cellId.substring(prefixSplit + 1);
}
if (prefix != null)
cellId = prefix + ':' + cellId;
Cell cell = alignment.getCell(cellId);
if (cell == null)
reporter.warn(
new IOMessageImpl("A cell referenced by a modifier could not be found", null));
return cell;
}
/**
* Generates a new prefix.
*
* @param prefixes the existing prefixes
* @return a new prefix
*/
private String generatePrefix(Set<String> prefixes) {
int prefixNumber = 1;
while (prefixes.contains("ba" + prefixNumber))
prefixNumber++;
return "ba" + prefixNumber;
}
}