/*
* Copyright 1998-2014 University Corporation for Atmospheric Research/Unidata
*
* Portions of this software were developed by the Unidata Program at the
* University Corporation for Atmospheric Research.
*
* Access and use of this software shall impose the following obligations
* and understandings on the user. The user is granted the right, without
* any fee or cost, to use, copy, modify, alter, enhance and distribute
* this software, and any derivative works thereof, and its supporting
* documentation for any purpose whatsoever, provided that this entire
* notice appears in all copies of the software, derivative works and
* supporting documentation. Further, UCAR requests that the user credit
* UCAR/Unidata in any publications that result from the use of this
* software or in any product that includes this software. The names UCAR
* and/or Unidata, however, may not be used in any advertising or publicity
* to endorse or promote any products or commercial entity unless specific
* written permission is obtained from UCAR/Unidata. The user also
* understands that UCAR/Unidata is not obligated to provide the user with
* any support, consulting, training or assistance of any kind with regard
* to the use, operation and performance of this software nor to provide
* the user with any updates, revisions, new versions or "bug fixes."
*
* THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
* INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
* FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
* NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
* WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package thredds.catalog.parser.jdom;
import thredds.featurecollection.FeatureCollectionConfig;
import thredds.util.PathAliasReplacement;
import thredds.catalog.*;
import thredds.crawlabledataset.*;
import thredds.crawlabledataset.sorter.LexigraphicByNameSorter;
import thredds.crawlabledataset.filter.*;
import thredds.cataloggen.ProxyDatasetHandler;
import thredds.cataloggen.DatasetEnhancer;
import thredds.cataloggen.CatalogRefExpander;
import thredds.cataloggen.datasetenhancer.RegExpAndDurationTimeCoverageEnhancer;
import thredds.cataloggen.inserter.SimpleLatestProxyDsHandler;
import thredds.cataloggen.inserter.LatestCompleteProxyDsHandler;
import org.jdom2.*;
import org.jdom2.input.*;
import org.jdom2.output.*;
import java.io.*;
import java.lang.reflect.Method;
import java.net.*;
import java.util.*;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.units.TimeDuration;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;
/**
* Inventory Catalog parser, version 1.0.
* Reads InvCatalog.xml files, constructs object representation.
*
* @author John Caron
*/
public class InvCatalogFactory10 implements InvCatalogConvertIF, MetadataConverterIF {
static private org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InvCatalogFactory10.class);
static public final Namespace defNS = Namespace.getNamespace(XMLEntityResolver.CATALOG_NAMESPACE_10);
static public final Namespace xlinkNS = Namespace.getNamespace("xlink", XMLEntityResolver.XLINK_NAMESPACE);
static public final Namespace ncmlNS = Namespace.getNamespace("ncml", XMLEntityResolver.NJ22_NAMESPACE);
static private boolean useBytesForDataSize = false;
static public void useBytesForDataSize(boolean b) {
useBytesForDataSize = b;
}
private InvCatalogFactory factory = null;
private String version = "1.0.1";
private boolean debugMetadataRead = false;
private List<PathAliasReplacement> dataRootLocAliasExpanders = Collections.emptyList();
public void setDataRootLocationAliasExpanders(List<PathAliasReplacement> dataRootLocAliasExpanders) {
if (dataRootLocAliasExpanders == null)
this.dataRootLocAliasExpanders = Collections.emptyList();
else
this.dataRootLocAliasExpanders = new ArrayList<>(dataRootLocAliasExpanders);
}
public List<PathAliasReplacement> getDataRootLocationAliasExpanders() {
return Collections.unmodifiableList(this.dataRootLocAliasExpanders);
}
private String expandAliasForPath(String location) {
for (PathAliasReplacement par : this.dataRootLocAliasExpanders) {
if (par.containsPathAlias(location)) {
return par.replacePathAlias(location);
}
}
return location;
}
private String expandAliasForCollectionSpec(String location) {
if (location == null) return null;
for (PathAliasReplacement par : this.dataRootLocAliasExpanders) {
String result = par.replaceIfMatch(location);
if (result != null) return result;
}
return location;
}
public InvCatalogImpl parseXML(InvCatalogFactory fac, org.jdom2.Document jdomDoc, URI uri) {
this.factory = fac;
return readCatalog(jdomDoc.getRootElement(), uri);
}
private Map<MetadataType, MetadataConverterIF> metadataHash = new HashMap<>(10);
public void registerMetadataConverter(MetadataType type, MetadataConverterIF converter) {
metadataHash.put(type, converter);
}
public void setVersion(String version) {
this.version = version;
}
/////////////////////////////////////////////////////////////////////////////
protected InvAccessImpl readAccess(InvDatasetImpl dataset, Element accessElem) {
String urlPath = accessElem.getAttributeValue("urlPath");
String serviceName = accessElem.getAttributeValue("serviceName");
String dataFormat = accessElem.getAttributeValue("dataFormat");
return new InvAccessImpl(dataset, urlPath, serviceName, null, dataFormat, readDataSize(accessElem));
}
protected InvCatalogImpl readCatalog(Element catalogElem, URI docBaseURI) {
String name = catalogElem.getAttributeValue("name");
String catSpecifiedBaseURL = catalogElem.getAttributeValue("base");
String expires = catalogElem.getAttributeValue("expires");
String version = catalogElem.getAttributeValue("version");
URI baseURI = docBaseURI;
if (catSpecifiedBaseURL != null) {
try {
baseURI = new URI(catSpecifiedBaseURL);
} catch (URISyntaxException e) {
logger.debug("readCatalog(): bad catalog specified base URI <" + catSpecifiedBaseURL + ">: " + e.getMessage(), e);
baseURI = docBaseURI;
}
}
InvCatalogImpl catalog = new InvCatalogImpl(name, version, makeDateType(expires, null, null), baseURI);
// read top-level services
java.util.List<Element> sList = catalogElem.getChildren("service", defNS);
for (Element e : sList) {
InvService s = readService(e, baseURI);
catalog.addService(s);
}
// read top-level properties
java.util.List<Element> pList = catalogElem.getChildren("property", defNS);
for (Element e : pList) {
InvProperty s = readProperty(e);
catalog.addProperty(s);
}
// read top-level dataroots
java.util.List<Element> rootList = catalogElem.getChildren("datasetRoot", defNS);
for (Element e : rootList) {
DataRootConfig root = readDatasetRoot(e);
catalog.addDatasetRoot(root);
}
// look for top-level dataset and catalogRefs elements (keep them in order)
java.util.List<Element> allChildren = catalogElem.getChildren();
for (Element e : allChildren) {
if (e.getName().equals("dataset")) {
catalog.addDataset(readDataset(catalog, null, e, baseURI));
} else if (e.getName().equals("featureCollection")) {
catalog.addDataset(readFeatureCollection(catalog, null, e, baseURI));
} else if (e.getName().equals("datasetScan")) {
catalog.addDataset(readDatasetScan(catalog, null, e, baseURI));
} else if (e.getName().equals("catalogRef")) {
catalog.addDataset(readCatalogRef(catalog, null, e, baseURI));
}
}
return catalog;
}
protected InvCatalogRef readCatalogRef(InvCatalogImpl cat, InvDatasetImpl parent, Element catRefElem, URI baseURI) {
String title = catRefElem.getAttributeValue("title", xlinkNS);
if (title == null) title = catRefElem.getAttributeValue("name");
String href = catRefElem.getAttributeValue("href", xlinkNS);
String useRemCatSerStr = catRefElem.getAttributeValue("useRemoteCatalogService");
Boolean useRemoteCatalogService = null;
if (useRemCatSerStr != null) useRemoteCatalogService = Boolean.parseBoolean(useRemCatSerStr);
InvCatalogRef catRef = new InvCatalogRef(parent, title, href, useRemoteCatalogService);
readDatasetInfo(cat, catRef, catRefElem, baseURI);
return catRef;
}
protected ThreddsMetadata.Contributor readContributor(Element elem) {
if (elem == null) return null;
return new ThreddsMetadata.Contributor(elem.getText(), elem.getAttributeValue("role"));
}
protected ThreddsMetadata.Vocab readControlledVocabulary(Element elem) {
if (elem == null) return null;
return new ThreddsMetadata.Vocab(elem.getText(), elem.getAttributeValue("vocabulary"));
}
// read a dataset element
protected InvDatasetImpl readDataset(InvCatalogImpl catalog, InvDatasetImpl parent, Element dsElem, URI base) {
// deal with aliases
String name = dsElem.getAttributeValue("name");
String alias = dsElem.getAttributeValue("alias");
if (alias != null) {
InvDatasetImpl ds = (InvDatasetImpl) catalog.findDatasetByID(alias);
if (ds == null) {
factory.appendErr(" ** Parse error: dataset named " + name + " has illegal alias = " + alias + "\n");
return null;
}
return new InvDatasetImplProxy(name, ds);
}
InvDatasetImpl dataset = new InvDatasetImpl(parent, name);
readDatasetInfo(catalog, dataset, dsElem, base);
if (InvCatalogFactory.debugXML) System.out.println(" Dataset added: " + dataset.dump());
return dataset;
}
protected void readDatasetInfo(InvCatalogImpl catalog, InvDatasetImpl dataset, Element dsElem, URI base) {
// read attributes
String authority = dsElem.getAttributeValue("authority");
String collectionTypeName = dsElem.getAttributeValue("collectionType");
String dataTypeName = dsElem.getAttributeValue("dataType");
String harvest = dsElem.getAttributeValue("harvest");
String id = dsElem.getAttributeValue("ID");
String serviceName = dsElem.getAttributeValue("serviceName");
String urlPath = dsElem.getAttributeValue("urlPath");
String restrictAccess = dsElem.getAttributeValue("restrictAccess");
FeatureType dataType = null;
if (dataTypeName != null) {
dataType = FeatureType.getType(dataTypeName.toUpperCase());
if (dataType == null) {
factory.appendWarning(" ** warning: non-standard data type = " + dataTypeName + "\n");
}
}
if (dataType != null)
dataset.setDataType(dataType);
if (serviceName != null)
dataset.setServiceName(serviceName);
if (urlPath != null)
dataset.setUrlPath(urlPath);
if (authority != null) dataset.setAuthority(authority);
if (id != null) dataset.setID(id);
if (harvest != null) dataset.setHarvest(harvest.equalsIgnoreCase("true"));
if (restrictAccess != null)
dataset.setResourceControl(restrictAccess);
if (collectionTypeName != null) {
CollectionType collectionType = CollectionType.findType(collectionTypeName);
if (collectionType == null) {
collectionType = CollectionType.getType(collectionTypeName);
factory.appendWarning(" ** warning: non-standard collection type = " + collectionTypeName + "\n");
}
dataset.setCollectionType(collectionType);
}
catalog.addDatasetByID(dataset); // need to do immed for alias processing
// look for services
java.util.List<Element> serviceList = dsElem.getChildren("service", defNS);
for (Element curElem : serviceList) {
InvService s = readService(curElem, base);
dataset.addService(s);
}
// look for direct thredds metadata (not inherited)
ThreddsMetadata tmg = dataset.getLocalMetadata();
readThreddsMetadata(catalog, dataset, dsElem, tmg);
// look for access elements
java.util.List<Element> aList = dsElem.getChildren("access", defNS);
for (Element e : aList) {
InvAccessImpl a = readAccess(dataset, e);
dataset.addAccess(a);
}
// look for ncml
Element ncmlElem = dsElem.getChild("netcdf", ncmlNS);
if (ncmlElem != null) {
ncmlElem.detach();
dataset.setNcmlElement(ncmlElem);
// System.out.println(" found ncml= "+ncmlElem);
}
// look for nested dataset and catalogRefs elements (keep them in order)
java.util.List<Element> allChildren = dsElem.getChildren();
for (Element e : allChildren) {
if (e.getName().equals("dataset")) {
InvDatasetImpl ds = readDataset(catalog, dataset, e, base);
if (ds != null)
dataset.addDataset(ds);
} else if (e.getName().equals("catalogRef")) {
InvDatasetImpl ds = readCatalogRef(catalog, dataset, e, base);
dataset.addDataset(ds);
} else if (e.getName().equals("datasetScan")) {
dataset.addDataset(readDatasetScan(catalog, dataset, e, base));
} else if (e.getName().equals("featureCollection")) {
InvDatasetImpl ds = readFeatureCollection(catalog, dataset, e, base);
if (ds != null)
dataset.addDataset(ds);
}
}
}
protected InvDatasetImpl readFeatureCollection(InvCatalogImpl catalog, InvDatasetImpl parent, Element dsElem, URI base) {
FeatureCollectionConfig config = FeatureCollectionReader.readFeatureCollection(dsElem);
config.spec = expandAliasForCollectionSpec(config.spec);
try {
// InvDatasetFeatureCollection ds = InvDatasetFeatureCollection.factory(parent, config);
Class c = this.getClass().getClassLoader().loadClass("thredds.catalog.InvDatasetFeatureCollection");
Method m = c.getMethod("factory", InvDatasetImpl.class, FeatureCollectionConfig.class);
InvDatasetImpl result = (InvDatasetImpl) m.invoke(null, parent, config);
if (result == null) {
logger.error("featureCollection " + config.collectionName + " has fatal error ");
return null;
}
// regular dataset elements
readDatasetInfo(catalog, result, dsElem, base);
return result;
} catch (Exception e) {
logger.error("featureCollection " + config.collectionName + " has fatal error, skipping ", e);
return null;
}
}
/* protected InvDatasetImpl readFeatureCollection(InvCatalogImpl catalog, InvDatasetImpl parent, Element dsElem, URI base) {
FeatureCollectionConfig config = FeatureCollectionReader.readFeatureCollection(dsElem);
config.spec = expandAliasForCollectionSpec(config.spec);
try {
InvDatasetFeatureCollection ds = InvDatasetFeatureCollection.factory(parent, config);
if (ds == null) {
logger.error("featureCollection " + config.collectionName + " has fatal error ");
return null;
}
// regular dataset elements
readDatasetInfo(catalog, ds, dsElem, base);
return ds;
} catch (Exception e) {
logger.error("featureCollection " + config.collectionName + " has fatal error, skipping ", e);
return null;
}
} */
// read a dataset scan element
protected InvDatasetScan readDatasetScan(InvCatalogImpl catalog, InvDatasetImpl parent, Element dsElem, URI base) {
InvDatasetScan datasetScan;
if (dsElem.getAttributeValue("dirLocation") == null) {
if (dsElem.getAttributeValue("location") == null) {
logger.error("readDatasetScan(): datasetScan has neither a \"location\" nor a \"dirLocation\" attribute.");
datasetScan = null;
} else {
return readDatasetScanNew(catalog, parent, dsElem, base);
}
} else {
String name = dsElem.getAttributeValue("name");
factory.appendWarning("**Warning: Dataset " + name + " using old form of DatasetScan (dirLocation instead of location)\n");
String path = dsElem.getAttributeValue("path");
String scanDir = expandAliasForPath(dsElem.getAttributeValue("dirLocation"));
String filter = dsElem.getAttributeValue("filter");
String addDatasetSizeString = dsElem.getAttributeValue("addDatasetSize");
String addLatest = dsElem.getAttributeValue("addLatest");
String sortOrderIncreasingString = dsElem.getAttributeValue("sortOrderIncreasing");
boolean sortOrderIncreasing = false;
if (sortOrderIncreasingString != null)
if (sortOrderIncreasingString.equalsIgnoreCase("true"))
sortOrderIncreasing = true;
boolean addDatasetSize = true;
if (addDatasetSizeString != null)
if (addDatasetSizeString.equalsIgnoreCase("false"))
addDatasetSize = false;
if (path != null) {
if (path.charAt(0) == '/') path = path.substring(1);
int last = path.length() - 1;
if (path.charAt(last) == '/') path = path.substring(0, last);
}
if (scanDir != null) {
int last = scanDir.length() - 1;
if (scanDir.charAt(last) != '/') scanDir = scanDir + '/';
}
Element atcElem = dsElem.getChild("addTimeCoverage", defNS);
String dsNameMatchPattern = null;
String startTimeSubstitutionPattern = null;
String duration = null;
if (atcElem != null) {
dsNameMatchPattern = atcElem.getAttributeValue("datasetNameMatchPattern");
startTimeSubstitutionPattern = atcElem.getAttributeValue("startTimeSubstitutionPattern");
duration = atcElem.getAttributeValue("duration");
}
try {
datasetScan = new InvDatasetScan(catalog, parent, name, path, scanDir, filter, addDatasetSize, addLatest, sortOrderIncreasing,
dsNameMatchPattern, startTimeSubstitutionPattern, duration);
readDatasetInfo(catalog, datasetScan, dsElem, base);
if (InvCatalogFactory.debugXML) System.out.println(" Dataset added: " + datasetScan.dump());
} catch (Exception e) {
logger.error("Reading DatasetScan", e);
datasetScan = null;
}
}
return datasetScan;
}
protected InvDatasetScan readDatasetScanNew(InvCatalogImpl catalog, InvDatasetImpl parent, Element dsElem, URI base) {
String name = dsElem.getAttributeValue("name");
String path = dsElem.getAttributeValue("path");
String locationOrg = dsElem.getAttributeValue("location");
String scanDir = expandAliasForPath(locationOrg);
if (scanDir.startsWith("${")) {
scanDir = expandAliasForPath(locationOrg); // debugging
}
// Read datasetConfig element
String configClassName = null;
Object configObj = null;
Element dsConfigElem = dsElem.getChild("crawlableDatasetImpl", defNS);
if (dsConfigElem != null) {
configClassName = dsConfigElem.getAttributeValue("className");
List children = dsConfigElem.getChildren();
if (children.size() == 1) {
configObj = children.get(0);
} else if (children.size() != 0) {
logger.warn("readDatasetScanNew(): content of datasetConfig element not a single element, using first element.");
configObj = children.get(0);
} else {
logger.debug("readDatasetScanNew(): datasetConfig element has no children.");
configObj = null;
}
}
// Read filter element
Element filterElem = dsElem.getChild("filter", defNS);
CrawlableDatasetFilter filter = null;
if (filterElem != null)
filter = readDatasetScanFilter(filterElem);
// Read identifier element
Element identifierElem = dsElem.getChild("addID", defNS);
CrawlableDatasetLabeler identifier = null;
if (identifierElem != null) {
identifier = readDatasetScanIdentifier(identifierElem);
}
// Read namer element
Element namerElem = dsElem.getChild("namer", defNS);
CrawlableDatasetLabeler namer = null;
if (namerElem != null) {
namer = readDatasetScanNamer(namerElem);
}
// Read sort element
Element sorterElem = dsElem.getChild("sort", defNS);
// By default, sort in decreasing lexigraphic order.
CrawlableDatasetSorter sorter = new LexigraphicByNameSorter(false);
if (sorterElem != null) {
sorter = readDatasetScanSorter(sorterElem);
}
// Read allProxies element (and addLatest element)
Element addLatestElem = dsElem.getChild("addLatest", defNS);
Element addProxiesElem = dsElem.getChild("addProxies", defNS);
Map<String, ProxyDatasetHandler> allProxyDsHandlers;
if (addLatestElem != null || addProxiesElem != null)
allProxyDsHandlers = readDatasetScanAddProxies(addProxiesElem, addLatestElem, catalog);
else
allProxyDsHandlers = new HashMap<>();
// Read addDatasetSize element.
Element addDsSizeElem = dsElem.getChild("addDatasetSize", defNS);
//boolean addDatasetSize = false; old way
//if ( addDsSizeElem != null )
// addDatasetSize = true;
boolean addDatasetSize = true;
if (addDsSizeElem != null) {
if (addDsSizeElem.getTextNormalize().equalsIgnoreCase("false"))
addDatasetSize = false;
}
// Read addTimeCoverage element.
List<DatasetEnhancer> childEnhancerList = new ArrayList<>();
Element addTimeCovElem = dsElem.getChild("addTimeCoverage", defNS);
if (addTimeCovElem != null) {
DatasetEnhancer addTimeCovEnhancer = readDatasetScanAddTimeCoverage(addTimeCovElem);
if (addTimeCovEnhancer != null)
childEnhancerList.add(addTimeCovEnhancer);
}
// Read datasetEnhancerImpl elements (user defined implementations of DatasetEnhancer)
List<Element> dsEnhancerElemList = dsElem.getChildren("datasetEnhancerImpl", defNS);
for (Element elem : dsEnhancerElemList) {
DatasetEnhancer o = readDatasetScanUserDefined(elem, DatasetEnhancer.class);
if (o != null)
childEnhancerList.add(o);
}
// Read catalogRefExpander element
// Element catRefExpanderElem = dsElem.getChild( "catalogRefExpander", defNS );
CatalogRefExpander catalogRefExpander = null;
// if ( catRefExpanderElem != null )
// {
// catalogRefExpander = readDatasetScanCatRefExpander( catRefExpanderElem );
// }
InvDatasetScan datasetScan;
try {
datasetScan = new InvDatasetScan(parent, name, path, scanDir,
configClassName, configObj,
filter, identifier, namer,
addDatasetSize, sorter, allProxyDsHandlers,
childEnhancerList,
catalogRefExpander);
readDatasetInfo(catalog, datasetScan, dsElem, base);
if (InvCatalogFactory.debugXML) System.out.println(" Dataset added: " + datasetScan.dump());
} catch (Exception e) {
logger.error("readDatasetScanNew(): failed to create DatasetScan", e);
datasetScan = null;
}
return datasetScan;
}
CrawlableDatasetFilter readDatasetScanFilter(Element filterElem) {
CrawlableDatasetFilter filter = null; //lastModifiedLimit
// Handle LastModifiedLimitFilter CrDsFilters.
Attribute lastModLimitAtt = filterElem.getAttribute("lastModifiedLimit");
if (lastModLimitAtt != null) {
long lastModLimit;
try {
lastModLimit = lastModLimitAtt.getLongValue();
} catch (DataConversionException e) {
String tmpMsg = "readDatasetScanFilter(): bad lastModifedLimit value <" + lastModLimitAtt.getValue() + ">, couldn't parse into long: " + e.getMessage();
factory.appendErr(tmpMsg);
logger.warn(tmpMsg);
return null;
}
return new LastModifiedLimitFilter(lastModLimit);
}
// Handle LogicalFilterComposer CrDsFilters.
String compType = filterElem.getAttributeValue("logicalComp");
if (compType != null) {
List filters = filterElem.getChildren("filter", defNS);
if (compType.equalsIgnoreCase("AND")) {
if (filters.size() != 2) {
String tmpMsg = "readDatasetScanFilter(): wrong number of filters <" + filters.size() + "> for AND (2 expected).";
factory.appendErr(tmpMsg);
logger.warn(tmpMsg);
return null;
}
filter = LogicalFilterComposer.getAndFilter(
readDatasetScanFilter((Element) filters.get(0)),
readDatasetScanFilter((Element) filters.get(1)));
} else if (compType.equalsIgnoreCase("OR")) {
if (filters.size() != 2) {
String tmpMsg = "readDatasetScanFilter(): wrong number of filters <" + filters.size() + "> for OR (2 expected).";
factory.appendErr(tmpMsg);
logger.warn(tmpMsg);
return null;
}
filter = LogicalFilterComposer.getOrFilter(
readDatasetScanFilter((Element) filters.get(0)),
readDatasetScanFilter((Element) filters.get(1)));
} else if (compType.equalsIgnoreCase("NOT")) {
if (filters.size() != 1) {
String tmpMsg = "readDatasetScanFilter(): wrong number of filters <" + filters.size() + "> for NOT (1 expected).";
factory.appendErr(tmpMsg);
logger.warn(tmpMsg);
return null;
}
filter = LogicalFilterComposer.getNotFilter(
readDatasetScanFilter((Element) filters.get(0)));
}
return filter;
}
// Handle user defined CrDsFilters.
Element userDefElem = filterElem.getChild("crawlableDatasetFilterImpl", defNS);
if (userDefElem != null) {
filter = (CrawlableDatasetFilter) readDatasetScanUserDefined(userDefElem, CrawlableDatasetFilter.class);
}
// Handle MultiSelectorFilter and contained Selectors.
else {
List<MultiSelectorFilter.Selector> selectorList = new ArrayList<>();
for (Element curElem : filterElem.getChildren()) {
String regExpAttVal = curElem.getAttributeValue("regExp");
String wildcardAttVal = curElem.getAttributeValue("wildcard");
String lastModLimitAttVal = curElem.getAttributeValue("lastModLimitInMillis");
if (regExpAttVal == null && wildcardAttVal == null && lastModLimitAttVal == null) {
// If no regExp or wildcard attributes, skip this selector.
logger.warn("readDatasetScanFilter(): no regExp, wildcard, or lastModLimitInMillis attribute in filter child <" + curElem.getName() + ">.");
} else {
// Determine if applies to atomic datasets, default true.
boolean atomic = true;
String atomicAttVal = curElem.getAttributeValue("atomic");
if (atomicAttVal != null) {
// If not "true", set to false.
if (!atomicAttVal.equalsIgnoreCase("true"))
atomic = false;
}
// Determine if applies to collection datasets, default false.
boolean collection = false;
String collectionAttVal = curElem.getAttributeValue("collection");
if (collectionAttVal != null) {
// If not "false", set to true.
if (!collectionAttVal.equalsIgnoreCase("false"))
collection = true;
}
// Determine if include or exclude selectors.
boolean includer = true;
if (curElem.getName().equals("exclude")) {
includer = false;
} else if (!curElem.getName().equals("include")) {
logger.warn("readDatasetScanFilter(): unhandled filter child <" + curElem.getName() + ">.");
continue;
}
// Determine if regExp or wildcard
if (regExpAttVal != null) {
selectorList.add(new MultiSelectorFilter.Selector(new RegExpMatchOnNameFilter(regExpAttVal), includer, atomic, collection));
} else if (wildcardAttVal != null) {
selectorList.add(new MultiSelectorFilter.Selector(new WildcardMatchOnNameFilter(wildcardAttVal), includer, atomic, collection));
} else if (lastModLimitAttVal != null) {
selectorList.add(new MultiSelectorFilter.Selector(new LastModifiedLimitFilter(Long.parseLong(lastModLimitAttVal)), includer, atomic, collection));
}
}
}
filter = new MultiSelectorFilter(selectorList);
}
return filter;
}
protected CrawlableDatasetLabeler readDatasetScanIdentifier(Element identifierElem) {
CrawlableDatasetLabeler identifier;
Element userDefElem = identifierElem.getChild("crawlableDatasetLabelerImpl", defNS);
if (userDefElem != null) {
identifier = (CrawlableDatasetLabeler) readDatasetScanUserDefined(userDefElem, CrawlableDatasetLabeler.class);
} else {
// Default is to add ID in standard way. Don't have alternates yet.
return null;
}
return identifier;
}
protected CrawlableDatasetLabeler readDatasetScanNamer(Element namerElem) {
CrawlableDatasetLabeler namer;
// Element userDefElem = namerElem.getChild( "crawlableDatasetLabelerImpl", defNS );
// if ( userDefElem != null )
// {
// namer = (CrawlableDatasetLabeler) readDatasetScanUserDefined( userDefElem, CrawlableDatasetLabeler.class );
// }
// else
// {
List<CrawlableDatasetLabeler> labelerList = new ArrayList<>();
for (Element curElem : namerElem.getChildren()) {
CrawlableDatasetLabeler curLabeler;
String regExp = curElem.getAttributeValue("regExp");
String replaceString = curElem.getAttributeValue("replaceString");
if (curElem.getName().equals("regExpOnName")) {
curLabeler = new RegExpAndReplaceOnNameLabeler(regExp, replaceString);
} else if (curElem.getName().equals("regExpOnPath")) {
curLabeler = new RegExpAndReplaceOnPathLabeler(regExp, replaceString);
} else {
logger.warn("readDatasetScanNamer(): unhandled namer child <" + curElem.getName() + ">.");
continue;
}
labelerList.add(curLabeler);
}
namer = new MultiLabeler(labelerList);
// }
return namer;
}
protected CrawlableDatasetSorter readDatasetScanSorter(Element sorterElem) {
CrawlableDatasetSorter sorter = null;
Element userDefElem = sorterElem.getChild("crawlableDatasetSorterImpl", defNS);
if (userDefElem != null) {
sorter = (CrawlableDatasetSorter) readDatasetScanUserDefined(userDefElem, CrawlableDatasetSorter.class);
} else {
Element lexSortElem = sorterElem.getChild("lexigraphicByName", defNS);
if (lexSortElem != null) {
boolean increasing;
String increasingString = lexSortElem.getAttributeValue("increasing");
increasing = increasingString.equalsIgnoreCase("true");
sorter = new LexigraphicByNameSorter(increasing);
}
}
return sorter;
}
protected Map<String, ProxyDatasetHandler> readDatasetScanAddProxies(Element addProxiesElem, Element addLatestElem, InvCatalogImpl catalog) {
Map<String, ProxyDatasetHandler> allProxyDsHandlers = new HashMap<>();
// Handle old "addLatest" elements.
if (addLatestElem != null) {
// Check for simpleLatest element.
Element simpleLatestElem = addLatestElem.getChild("simpleLatest", defNS);
// Get a SimpleLatestDsHandler, use default values if element is null.
ProxyDatasetHandler pdh = readDatasetScanAddLatest(simpleLatestElem, catalog);
if (pdh != null)
allProxyDsHandlers.put(pdh.getProxyDatasetName(), pdh);
}
// Handle all "addProxies" elements.
if (addProxiesElem != null) {
for (Element curChildElem : addProxiesElem.getChildren()) {
ProxyDatasetHandler curPdh;
// Handle "simpleLatest" child elements.
if (curChildElem.getName().equals("simpleLatest")
&& curChildElem.getNamespace().equals(defNS)) {
curPdh = readDatasetScanAddLatest(curChildElem, catalog);
}
// Handle "latestComplete" child elements.
else if (curChildElem.getName().equals("latestComplete")
&& curChildElem.getNamespace().equals(defNS)) {
// Get latest name.
String latestName = curChildElem.getAttributeValue("name");
if (latestName == null) {
logger.warn("readDatasetScanAddProxies(): unnamed latestComplete, skipping.");
continue;
}
// Does latest go on top or bottom of list.
Attribute topAtt = curChildElem.getAttribute("top");
boolean latestOnTop = true;
if (topAtt != null) {
try {
latestOnTop = topAtt.getBooleanValue();
} catch (DataConversionException e) {
latestOnTop = true;
}
}
// Get the latest service name.
String serviceName = curChildElem.getAttributeValue("serviceName");
if (serviceName == null) {
logger.warn("readDatasetScanAddProxies(): no service name given in latestComplete.");
continue;
}
InvService service = catalog.findService(serviceName);
if (service == null) {
logger.warn("readDatasetScanAddProxies(): named service <" + serviceName + "> not found.");
continue;
}
// Get lastModifed limit.
String lastModLimitVal = curChildElem.getAttributeValue("lastModifiedLimit");
long lastModLimit;
if (lastModLimitVal == null)
lastModLimit = 60; // Default to one hour
else
lastModLimit = Long.parseLong(lastModLimitVal);
// Get isResolver.
String isResolverString = curChildElem.getAttributeValue("isResolver");
boolean isResolver = true;
if (isResolverString != null)
if (isResolverString.equalsIgnoreCase("false"))
isResolver = false;
// Build the SimpleLatestProxyDsHandler and add to map.
curPdh = new LatestCompleteProxyDsHandler(latestName, latestOnTop, service, isResolver, lastModLimit);
} else {
curPdh = null;
// @todo Deal with allowing user defined inserters
//Element userDefElem = addLatestElem.getChild( "proxyDatasetHandlerImpl", defNS );
}
// Add current proxy dataset handler to map if name is not already in map.
if (curPdh != null) {
if (allProxyDsHandlers.containsKey(curPdh.getProxyDatasetName())) {
logger.warn("readDatasetScanAddProxies(): proxy map already contains key <" + curPdh.getProxyDatasetName() + ">, skipping.");
continue;
}
allProxyDsHandlers.put(curPdh.getProxyDatasetName(), curPdh);
}
}
}
return allProxyDsHandlers;
}
/**
* Return a SimpleLatestProxyDsHandler, use default values if element is null.
*
* @param simpleLatestElem the simpleLatest element
* @param catalog the catalog containing the simpleLatest element.
* @return a SimpleLatestProxyDsHandler
*/
private ProxyDatasetHandler readDatasetScanAddLatest(Element simpleLatestElem, InvCatalogImpl catalog) {
// Default values is simpleLatestElem is null.
ProxyDatasetHandler latestAdder = null;
String latestName = "latest.xml";
boolean latestOnTop = true;
String latestServiceName = "latest";
boolean isResolver = true;
// If simpleLatestElem exists, read values.
if (simpleLatestElem != null) {
// Get latest name.
String tmpLatestName = simpleLatestElem.getAttributeValue("name");
if (tmpLatestName != null)
latestName = tmpLatestName;
// Does latest go on top or bottom of list.
Attribute topAtt = simpleLatestElem.getAttribute("top");
if (topAtt != null) {
try {
latestOnTop = topAtt.getBooleanValue();
} catch (DataConversionException e) {
latestOnTop = true;
}
}
// Get the latest service name.
String tmpLatestServiceName = simpleLatestElem.getAttributeValue("serviceName");
if (tmpLatestServiceName != null)
latestServiceName = tmpLatestServiceName;
// Get isResolver.
String isResolverString = simpleLatestElem.getAttributeValue("isResolver");
if (isResolverString != null)
if (isResolverString.equalsIgnoreCase("false"))
isResolver = false;
}
// Build the SimpleLatestProxyDsHandler
InvService service = catalog.findService(latestServiceName);
if (service == null)
logger.warn("readDatasetScanAddLatest(): named service <" + latestServiceName + "> not found.");
else
latestAdder = new SimpleLatestProxyDsHandler(latestName, latestOnTop, service, isResolver);
return latestAdder;
}
// protected CatalogRefExpander readDatasetScanCatRefExpander( Element catRefExpanderElem )
// {
//
// }
protected DatasetEnhancer readDatasetScanAddTimeCoverage(Element addTimeCovElem) {
DatasetEnhancer timeCovEnhancer = null;
String matchName = addTimeCovElem.getAttributeValue("datasetNameMatchPattern");
String matchPath = addTimeCovElem.getAttributeValue("datasetPathMatchPattern");
String subst = addTimeCovElem.getAttributeValue("startTimeSubstitutionPattern");
String duration = addTimeCovElem.getAttributeValue("duration");
if (matchName != null && subst != null && duration != null) {
timeCovEnhancer = RegExpAndDurationTimeCoverageEnhancer
.getInstanceToMatchOnDatasetName(matchName, subst, duration);
} else if (matchPath != null && subst != null && duration != null) {
timeCovEnhancer = RegExpAndDurationTimeCoverageEnhancer
.getInstanceToMatchOnDatasetPath(matchPath, subst, duration);
}
return timeCovEnhancer;
}
private DatasetEnhancer readDatasetScanUserDefined(Element userDefElem, Class targetClass) {
String className = userDefElem.getAttributeValue("className");
Element configElem;
List childrenElemList = userDefElem.getChildren();
if (childrenElemList.size() == 1) {
configElem = (Element) childrenElemList.get(0);
} else if (childrenElemList.size() != 0) {
logger.warn("readDatasetScanUserDefined(): config XML not a single element, using first element.");
configElem = (Element) childrenElemList.get(0);
} else {
logger.debug("readDatasetScanUserDefined(): no config XML elements.");
configElem = null;
}
try {
// Get the Class instance for desired targetClass implementation.
Class requestedClass = Class.forName(className);
// Check that the requested Class is a target Class.
if (!targetClass.isAssignableFrom(requestedClass)) {
throw new IllegalArgumentException("Requested class <" + className + "> not an implementation of " + targetClass.getName() + ".");
}
// Instantiate the desired Object using that classes constructor with a
// single Object argument.
Class[] argTypes = {Object.class};
Object[] args = {configElem};
Constructor constructor = requestedClass.getConstructor(argTypes);
return (DatasetEnhancer) constructor.newInstance(args);
} catch (ClassNotFoundException e) {
logger.warn("readDatasetScanUserDefined(): exception creating user defined object <" + className + ">", e);
return null;
} catch (NoSuchMethodException e) {
logger.warn("readDatasetScanUserDefined(): exception creating user defined object <" + className + ">", e);
return null;
} catch (InstantiationException e) {
logger.warn("readDatasetScanUserDefined(): exception creating user defined object <" + className + ">", e);
return null;
} catch (IllegalAccessException e) {
logger.warn("readDatasetScanUserDefined(): exception creating user defined object <" + className + ">", e);
return null;
} catch (InvocationTargetException e) {
logger.warn("readDatasetScanUserDefined(): exception creating user defined object <" + className + ">", e);
return null;
}
}
protected DataRootConfig readDatasetRoot(Element s) {
String path = s.getAttributeValue("path");
String dirLocation = s.getAttributeValue("location");
if (dirLocation == null)
dirLocation = s.getAttributeValue("dirLocation");
dirLocation = expandAliasForPath(dirLocation);
if (path != null) {
if (path.charAt(0) == '/') path = path.substring(1);
int last = path.length() - 1;
if (path.charAt(last) == '/') path = path.substring(0, last);
}
if (dirLocation != null) {
int last = dirLocation.length() - 1;
if (dirLocation.charAt(last) != '/') dirLocation = dirLocation + '/';
}
return new DataRootConfig(path, dirLocation, s.getAttributeValue("cache"));
}
protected DateType readDate(Element elem) {
if (elem == null) return null;
String format = elem.getAttributeValue("format");
String type = elem.getAttributeValue("type");
return makeDateType(elem.getText(), format, type);
}
protected DateType makeDateType(String text, String format, String type) {
if (text == null) return null;
try {
return new DateType(text, format, type);
} catch (java.text.ParseException e) {
factory.appendErr(" ** Parse error: Bad date format = " + text + "\n");
return null;
}
}
protected TimeDuration readDuration(Element elem) {
if (elem == null) return null;
String text = null;
try {
text = elem.getText();
return new TimeDuration(text);
} catch (java.text.ParseException e) {
factory.appendErr(" ** Parse error: Bad duration format = " + text + "\n");
return null;
}
}
protected InvDocumentation readDocumentation(InvCatalog cat, Element s) {
String href = s.getAttributeValue("href", xlinkNS);
String title = s.getAttributeValue("title", xlinkNS);
String type = s.getAttributeValue("type"); // not XLink type
String content = s.getTextNormalize();
URI uri = null;
if (href != null) {
try {
uri = cat.resolveUri(href);
} catch (Exception e) {
factory.appendErr(" ** Invalid documentation href = " + href + " " + e.getMessage() + "\n");
}
}
InvDocumentation doc = new InvDocumentation(href, uri, title, type, content);
// LOOK XHTML ?? !!
if (InvCatalogFactory.debugXML) System.out.println(" Documentation added: " + doc);
return doc;
}
protected double readDouble(Element elem) {
if (elem == null) return Double.NaN;
String text = elem.getText();
try {
return Double.parseDouble(text);
} catch (NumberFormatException e) {
factory.appendErr(" ** Parse error: Bad double format = " + text + "\n");
return Double.NaN;
}
}
protected ThreddsMetadata.GeospatialCoverage readGeospatialCoverage(Element gcElem) {
if (gcElem == null) return null;
String zpositive = gcElem.getAttributeValue("zpositive");
ThreddsMetadata.Range northsouth = readGeospatialRange(gcElem.getChild("northsouth", defNS), CDM.LAT_UNITS);
ThreddsMetadata.Range eastwest = readGeospatialRange(gcElem.getChild("eastwest", defNS), CDM.LON_UNITS);
ThreddsMetadata.Range updown = readGeospatialRange(gcElem.getChild("updown", defNS), "m");
// look for names
List<ThreddsMetadata.Vocab> names = new ArrayList<>();
java.util.List<Element> list = gcElem.getChildren("name", defNS);
for (Element e : list) {
ThreddsMetadata.Vocab name = readControlledVocabulary(e);
names.add(name);
}
return new ThreddsMetadata.GeospatialCoverage(eastwest, northsouth, updown, names, zpositive);
}
protected ThreddsMetadata.Range readGeospatialRange(Element spElem, String defUnits) {
if (spElem == null) return null;
double start = readDouble(spElem.getChild("start", defNS));
double size = readDouble(spElem.getChild("size", defNS));
double resolution = readDouble(spElem.getChild("resolution", defNS));
String units = spElem.getChildText("units", defNS);
if (units == null) units = defUnits;
return new ThreddsMetadata.Range(start, size, resolution, units);
}
protected InvMetadata readMetadata(InvCatalog catalog, InvDatasetImpl dataset, Element mdataElement) {
// there are 6 cases to deal with: threddsNamespace vs not & inline vs Xlink & hasConverter or not
// (the hasConverter only applies when its not threddsNamespace, giving 6 cases)
// this factory is the converter for threddsNamespace metadata
// and also handles non-threddsNamespace when there is no converter, in which case it just
// propagates the inline dom elements
// figure out the namespace
Namespace namespace;
List inlineElements = mdataElement.getChildren();
if (inlineElements.size() > 0) // look at the namespace of the children, if they exist
namespace = ((Element) inlineElements.get(0)).getNamespace();
else
namespace = mdataElement.getNamespace(); // will be thredds
String mtype = mdataElement.getAttributeValue("metadataType");
String href = mdataElement.getAttributeValue("href", xlinkNS);
String title = mdataElement.getAttributeValue("title", xlinkNS);
String inheritedS = mdataElement.getAttributeValue("inherited");
boolean inherited = (inheritedS != null) && inheritedS.equalsIgnoreCase("true");
boolean isThreddsNamespace = ((mtype == null) || mtype.equalsIgnoreCase("THREDDS")) &&
namespace.getURI().equals(XMLEntityResolver.CATALOG_NAMESPACE_10);
// see if theres a converter for it.
MetadataConverterIF metaConverter = factory.getMetadataConverter(namespace.getURI());
if (metaConverter == null) metaConverter = factory.getMetadataConverter(mtype);
if (metaConverter != null) {
if (debugMetadataRead) System.out.println("found factory for metadata type = " + mtype + " namespace = " +
namespace + "=" + metaConverter.getClass().getName());
// see if theres any inline content
Object contentObj;
if (inlineElements.size() > 0) {
contentObj = metaConverter.readMetadataContent(dataset, mdataElement);
return new InvMetadata(dataset, mtype, namespace.getURI(), namespace.getPrefix(),
inherited, false, metaConverter, contentObj);
} else { // otherwise it must be an Xlink; defer reading
return new InvMetadata(dataset, href, title, mtype, namespace.getURI(),
namespace.getPrefix(), inherited, false, metaConverter);
}
}
// the case where its not ThreddsMetadata, but theres no converter
if (!isThreddsNamespace) {
if (inlineElements.size() > 0) {
// just hold onto the jdom elements as the "content" LOOK should be DOM?
return new InvMetadata(dataset, mtype, namespace.getURI(), namespace.getPrefix(),
inherited, false, this, mdataElement);
} else { // otherwise it must be an Xlink, never read
return new InvMetadata(dataset, href, title, mtype, namespace.getURI(),
namespace.getPrefix(), inherited, false, null);
}
}
// the case where its ThreddsMetadata
if (inlineElements.size() > 0) {
ThreddsMetadata tmg = new ThreddsMetadata(false);
readThreddsMetadata(catalog, dataset, mdataElement, tmg);
return new InvMetadata(dataset, mtype, namespace.getURI(), namespace.getPrefix(),
inherited, true, this, tmg);
} else { // otherwise it must be an Xlink; defer reading
return new InvMetadata(dataset, href, title, mtype, namespace.getURI(),
namespace.getPrefix(), inherited, true, this);
}
}
/* MetadataConverterIF */
public Object readMetadataContent(InvDataset dataset, org.jdom2.Element mdataElement) {
InvMetadata m = readMetadata(dataset.getParentCatalog(), (InvDatasetImpl) dataset, mdataElement);
return m.getThreddsMetadata();
}
private SAXBuilder saxBuilder;
private Element readContentFromURL(java.net.URI uri) throws java.io.IOException {
if (saxBuilder == null) saxBuilder = new SAXBuilder();
Document doc;
try {
doc = saxBuilder.build(uri.toURL());
} catch (JDOMException e) {
throw new IOException(e.getMessage());
}
return doc.getRootElement();
}
// this is only called for ThredddsMetadata
public Object readMetadataContentFromURL(InvDataset dataset, java.net.URI uri) throws java.io.IOException {
Element elem = readContentFromURL(uri);
Object contentObject = readMetadataContent(dataset, elem);
if (debugMetadataRead) System.out.println(" convert to " + contentObject.getClass().getName());
return contentObject;
}
/* open and read the referenced catalog XML
if (debugMetadataRead) System.out.println(" readMetadataContentFromURL = " + url);
org.w3c.dom.Element mdataElement = factory.readOtherXML( url);
if (mdataElement == null) {
factory.appendErr(" ** failed to read thredds metadata at = "+url+" for dataset"+dataset.getName()+"\n");
return null;
}
Object contentObject = readMetadataContent( dataset, mdataElement);
if (debugMetadataRead) System.out.println(" convert to " + contentObject.getClass().getName());
return contentObject; */
// dummy LOOK
public boolean validateMetadataContent(Object contentObject, StringBuilder out) {
return true;
}
public void addMetadataContent(org.jdom2.Element mdataElement, Object contentObject) {
}
protected InvProperty readProperty(Element s) {
String name = s.getAttributeValue("name");
String value = s.getAttributeValue("value");
return new InvProperty(name, value);
}
protected ThreddsMetadata.Source readSource(Element elem) {
if (elem == null) return null;
ThreddsMetadata.Vocab name = readControlledVocabulary(elem.getChild("name", defNS));
Element contact = elem.getChild("contact", defNS);
if (contact == null) {
factory.appendErr(" ** Parse error: Missing contact element in = " + elem.getName() + "\n");
return null;
}
return new ThreddsMetadata.Source(name, contact.getAttributeValue("url"), contact.getAttributeValue("email"));
}
protected InvService readService(Element s, URI baseURI) {
String name = s.getAttributeValue("name");
String type = s.getAttributeValue("serviceType");
String serviceBase = s.getAttributeValue("base");
String suffix = s.getAttributeValue("suffix");
String desc = s.getAttributeValue("desc");
InvService service = new InvService(name, type, serviceBase, suffix, desc);
java.util.List<Element> propertyList = s.getChildren("property", defNS);
for (Element e : propertyList) {
InvProperty p = readProperty(e);
service.addProperty(p);
}
java.util.List<Element> rootList = s.getChildren("datasetRoot", defNS);
for (Element e : rootList) {
InvProperty root = readDatasetRoot(e);
service.addDatasetRoot(root);
}
// nested services
java.util.List<Element> serviceList = s.getChildren("service", defNS);
for (Element e : serviceList) {
InvService ss = readService(e, baseURI);
service.addService(ss);
}
if (InvCatalogFactory.debugXML) System.out.println(" Service added: " + service);
return service;
}
protected double readDataSize(Element parent) {
Element elem = parent.getChild("dataSize", defNS);
if (elem == null) return Double.NaN;
double size;
String sizeS = elem.getText();
try {
size = Double.parseDouble(sizeS);
} catch (NumberFormatException e) {
factory.appendErr(" ** Parse error: Bad double format in size element = " + sizeS + "\n");
return Double.NaN;
}
String units = elem.getAttributeValue("units");
char c = Character.toUpperCase(units.charAt(0));
if (c == 'K') size *= 1000;
else if (c == 'M') size *= 1000 * 1000;
else if (c == 'G') size *= 1000 * 1000 * 1000;
else if (c == 'T') size *= 1000.0 * 1000 * 1000 * 1000;
else if (c == 'P') size *= 1000.0 * 1000 * 1000 * 1000 * 1000;
return size;
}
protected DateRange readTimeCoverage(Element tElem) {
if (tElem == null) return null;
DateType start = readDate(tElem.getChild("start", defNS));
DateType end = readDate(tElem.getChild("end", defNS));
TimeDuration duration = readDuration(tElem.getChild("duration", defNS));
TimeDuration resolution = readDuration(tElem.getChild("resolution", defNS));
try {
return new DateRange(start, end, duration, resolution);
} catch (java.lang.IllegalArgumentException e) {
factory.appendWarning(" ** warning: TimeCoverage error = " + e.getMessage() + "\n");
return null;
}
}
protected void readThreddsMetadata(InvCatalog catalog, InvDatasetImpl dataset, Element parent, ThreddsMetadata tmg) {
List<Element> list;
// look for creators - kind of a Source
list = parent.getChildren("creator", defNS);
for (Element e : list) {
tmg.addCreator(readSource(e));
}
// look for contributors
list = parent.getChildren("contributor", defNS);
for (Element e : list) {
tmg.addContributor(readContributor(e));
}
// look for dates
list = parent.getChildren("date", defNS);
for (Element e : list) {
DateType d = readDate(e);
tmg.addDate(d);
}
// look for documentation
list = parent.getChildren("documentation", defNS);
for (Element e : list) {
InvDocumentation doc = readDocumentation(catalog, e);
tmg.addDocumentation(doc);
}
// look for keywords - kind of a controlled vocabulary
list = parent.getChildren("keyword", defNS);
for (Element e : list) {
tmg.addKeyword(readControlledVocabulary(e));
}
// look for metadata
java.util.List<Element> mList = parent.getChildren("metadata", defNS);
for (Element e : mList) {
InvMetadata m = readMetadata(catalog, dataset, e);
if (m != null) {
tmg.addMetadata(m);
}
}
// look for projects - kind of a controlled vocabulary
list = parent.getChildren("project", defNS);
for (Element e : list) {
tmg.addProject(readControlledVocabulary(e));
}
// look for properties
list = parent.getChildren("property", defNS);
for (Element e : list) {
InvProperty p = readProperty(e);
tmg.addProperty(p);
}
// look for publishers - kind of a Source
list = parent.getChildren("publisher", defNS);
for (Element e : list) {
tmg.addPublisher(readSource(e));
}
// look for variables
list = parent.getChildren("variables", defNS);
for (Element e : list) {
ThreddsMetadata.Variables vars = readVariables(catalog, dataset, e);
tmg.addVariables(vars);
}
// can only be one each of these kinds
ThreddsMetadata.GeospatialCoverage gc = readGeospatialCoverage(parent.getChild("geospatialCoverage", defNS));
if (gc != null) tmg.setGeospatialCoverage(gc);
DateRange tc = readTimeCoverage(parent.getChild("timeCoverage", defNS));
if (tc != null) tmg.setTimeCoverage(tc);
Element serviceNameElem = parent.getChild("serviceName", defNS);
if (serviceNameElem != null) tmg.setServiceName(serviceNameElem.getText());
Element authElem = parent.getChild("authority", defNS);
if (authElem != null) tmg.setAuthority(authElem.getText());
Element dataTypeElem = parent.getChild("dataType", defNS);
if (dataTypeElem != null) {
String dataTypeName = dataTypeElem.getText();
if ((dataTypeName != null) && (dataTypeName.length() > 0)) {
FeatureType dataType = FeatureType.getType(dataTypeName.toUpperCase());
if (dataType == null) {
factory.appendWarning(" ** warning: non-standard data type = " + dataTypeName + "\n");
}
tmg.setDataType(dataType);
}
}
Element dataFormatElem = parent.getChild("dataFormat", defNS);
if (dataFormatElem != null) {
String dataFormatTypeName = dataFormatElem.getText();
if ((dataFormatTypeName != null) && (dataFormatTypeName.length() > 0)) {
DataFormatType dataFormatType = DataFormatType.findType(dataFormatTypeName);
if (dataFormatType == null) {
dataFormatType = DataFormatType.getType(dataFormatTypeName);
factory.appendWarning(" ** warning: non-standard dataFormat type = " + dataFormatTypeName + "\n");
}
tmg.setDataFormatType(dataFormatType);
}
}
double size = readDataSize(parent);
if (!Double.isNaN(size))
tmg.setDataSize(size);
}
protected ThreddsMetadata.Variable readVariable(Element varElem) {
if (varElem == null) return null;
String name = varElem.getAttributeValue("name");
String desc = varElem.getText();
String vocabulary_name = varElem.getAttributeValue("vocabulary_name");
String units = varElem.getAttributeValue("units");
String id = varElem.getAttributeValue("vocabulary_id");
return new ThreddsMetadata.Variable(name, desc, vocabulary_name, units, id);
}
protected ThreddsMetadata.Variables readVariables(InvCatalog cat, InvDataset ds, Element varsElem) {
if (varsElem == null) return null;
String vocab = varsElem.getAttributeValue("vocabulary");
String vocabHref = varsElem.getAttributeValue("href", xlinkNS);
URI vocabUri = null;
if (vocabHref != null) {
try {
vocabUri = cat.resolveUri(vocabHref);
} catch (Exception e) {
factory.appendErr(" ** Invalid Variables vocabulary URI = " + vocabHref + " " + e.getMessage() + "\n");
}
}
java.util.List<Element> vlist = varsElem.getChildren("variable", defNS);
String mapHref = null;
URI mapUri = null;
Element map = varsElem.getChild("variableMap", defNS);
if (map != null) {
mapHref = map.getAttributeValue("href", xlinkNS);
try {
mapUri = cat.resolveUri(mapHref);
} catch (Exception e) {
factory.appendErr(" ** Invalid Variables map URI = " + mapHref + " " + e.getMessage() + "\n");
}
}
if ((mapUri != null) && vlist.size() > 0) { // cant do both
factory.appendErr(" ** Catalog error: cant have variableMap and variable in same element (dataset = " +
ds.getName() + "\n");
mapUri = null;
}
ThreddsMetadata.Variables variables = new ThreddsMetadata.Variables(vocab, vocabHref, vocabUri, mapHref, mapUri);
for (Element e : vlist) {
ThreddsMetadata.Variable v = readVariable(e);
variables.addVariable(v);
}
// read in variable map LOOK: would like to defer
if (mapUri != null) {
Element varsElement;
try {
varsElement = readContentFromURL(mapUri);
List<Element> list = varsElement.getChildren("variable", defNS);
for (Element e : list) {
ThreddsMetadata.Variable v = readVariable(e);
variables.addVariable(v);
}
} catch (IOException e) {
logger.warn("Failure reading vaiable mapUri ", e);
}
/*org.w3c.dom.Element domElement = factory.readOtherXML(mapUri);
if (domElement != null) {
Element varsElement = toJDOM(domElement);
List list = varsElement.getChildren("variable", defNS);
for (int j = 0; j < list.size(); j++) {
ThreddsMetadata.Variable v = readVariable( (Element) list.get(j));
variables.addVariable(v);
}
} */
}
return variables;
}
/************************************************************************/
// Writing XML from objects
/**
* Write the catalog as an XML document to the specified stream.
*
* @param catalog write this catalog
* @param os write to this OutputStream
* @param raw write raw file if true (for server configuration)
* @throws IOException
*/
public void writeXML(InvCatalogImpl catalog, OutputStream os, boolean raw) throws IOException {
this.raw = raw;
writeXML(catalog, os);
this.raw = false;
}
private boolean raw = false;
/**
* Write the catalog as an XML document to the specified stream.
*
* @param catalog write this catalog
* @param os write to this OutputStream
* @throws IOException
*/
public void writeXML(InvCatalogImpl catalog, OutputStream os) throws IOException {
// Output the document, use standard formatter
//XMLOutputter fmt = new XMLOutputter();
//fmt.setNewlines(true);
//fmt.setIndent(" ");
//fmt.setTrimAllWhite( true);
XMLOutputter fmt = new XMLOutputter(org.jdom2.output.Format.getPrettyFormat()); // LOOK maybe compact ??
fmt.output(writeCatalog(catalog), os);
}
public Document writeCatalog(InvCatalogImpl cat) {
Element rootElem = new Element("catalog", defNS);
Document doc = new Document(rootElem);
// attributes
if (cat.getName() != null)
rootElem.setAttribute("name", cat.getName());
rootElem.setAttribute("version", version);
rootElem.addNamespaceDeclaration(xlinkNS);
if (cat.getExpires() != null)
rootElem.setAttribute("expires", cat.getExpires().toString());
// services
Iterator iter = cat.getServices().iterator();
while (iter.hasNext()) {
InvService service = (InvService) iter.next();
rootElem.addContent(writeService(service));
}
// dataset roots
if (raw) {
iter = cat.getDatasetRoots().iterator();
while (iter.hasNext()) {
InvProperty p = (InvProperty) iter.next();
rootElem.addContent(writeDatasetRoot(p));
}
}
// properties
iter = cat.getProperties().iterator();
while (iter.hasNext()) {
InvProperty p = (InvProperty) iter.next();
rootElem.addContent(writeProperty(p));
}
// datasets
iter = cat.getDatasets().iterator();
while (iter.hasNext()) {
InvDatasetImpl ds = (InvDatasetImpl) iter.next();
if (ds instanceof InvDatasetScan)
rootElem.addContent(writeDatasetScan((InvDatasetScan) ds));
else if (ds instanceof InvCatalogRef)
rootElem.addContent(writeCatalogRef((InvCatalogRef) ds));
else
rootElem.addContent(writeDataset(ds));
}
return doc;
}
private Element writeAccess(InvAccessImpl access) {
Element accessElem = new Element("access", defNS);
accessElem.setAttribute("urlPath", access.getUrlPath());
if (access.getServiceName() != null)
accessElem.setAttribute("serviceName", access.getServiceName());
if (access.getDataFormatType() != null)
accessElem.setAttribute("dataFormat", access.getDataFormatType().toString());
if (access.hasDataSize())
accessElem.addContent(writeDataSize(access.getDataSize()));
return accessElem;
}
private Element writeCatalogRef(InvCatalogRef catRef) {
Element catrefElem = new Element("catalogRef", defNS);
catrefElem.setAttribute("href", catRef.getXlinkHref(), xlinkNS);
String name = catRef.getName() == null ? "" : catRef.getName();
catrefElem.setAttribute("title", name, xlinkNS);
if (catRef.getID() != null)
catrefElem.setAttribute("ID", catRef.getID());
if (catRef.getRestrictAccess() != null)
catrefElem.setAttribute("restrictAccess", catRef.getRestrictAccess());
catrefElem.setAttribute("name", "");
/* List list = catRef.getDocumentation();
for (int j=0; j< list.size(); j++) {
InvDocumentation doc = (InvDocumentation) list.get(j);
catrefElem.addContent( writeDocumentation(doc, "documentation"));
} */
return catrefElem;
}
protected Element writeContributor(ThreddsMetadata.Contributor c) {
Element elem = new Element("contributor", defNS);
if (c.getRole() != null)
elem.setAttribute("role", c.getRole());
elem.setText(c.getName());
return elem;
}
private Element writeControlledVocabulary(ThreddsMetadata.Vocab v, String name) {
Element elem = new Element(name, defNS);
if (v.getVocabulary() != null)
elem.setAttribute("vocabulary", v.getVocabulary());
elem.addContent(v.getText());
return elem;
}
private Element writeDataset(InvDatasetImpl ds) {
Element dsElem = new Element("dataset", defNS);
if (ds instanceof InvDatasetImplProxy) {
dsElem.setAttribute("name", ((InvDatasetImplProxy) ds).getAliasName());
dsElem.setAttribute("alias", ds.getID());
return dsElem;
}
writeDatasetInfo(ds, dsElem, true, raw);
return dsElem;
}
private Element writeDatasetRoot(InvProperty prop) {
Element drootElem = new Element("datasetRoot", defNS);
drootElem.setAttribute("path", prop.getName());
drootElem.setAttribute("location", prop.getValue());
return drootElem;
}
private Element writeDatasetScan(InvDatasetScan ds) {
Element dsElem;
if (raw) {
// Setup datasetScan element
dsElem = new Element("datasetScan", defNS);
writeDatasetInfo(ds, dsElem, false, true);
dsElem.setAttribute("path", ds.getPath());
dsElem.setAttribute("location", ds.getScanLocation());
// Write datasetConfig element
if (ds.getCrDsClassName() != null) {
Element configElem = new Element("crawlableDatasetImpl", defNS);
configElem.setAttribute("className", ds.getCrDsClassName());
if (ds.getCrDsConfigObj() != null) {
if (ds.getCrDsConfigObj() instanceof Element) {
configElem.addContent((Element) ds.getCrDsConfigObj());
}
}
}
// Write filter element
if (ds.getFilter() != null)
dsElem.addContent(writeDatasetScanFilter(ds.getFilter()));
// Write addID element
//if ( ds.getIdentifier() != null )
dsElem.addContent(writeDatasetScanIdentifier(ds.getIdentifier()));
// Write namer element
if (ds.getNamer() != null)
dsElem.addContent(writeDatasetScanNamer(ds.getNamer()));
// Write sort element
if (ds.getSorter() != null)
dsElem.addContent(writeDatasetScanSorter(ds.getSorter()));
// Write addProxy element (and old addLatest element)
if (!ds.getProxyDatasetHandlers().isEmpty())
dsElem.addContent(writeDatasetScanAddProxies(ds.getProxyDatasetHandlers()));
// Write addDatasetSize element
if (ds.getAddDatasetSize())
dsElem.addContent(new Element("addDatasetSize", defNS));
// Write addTimeCoverage and datasetEnhancerImpl elements
if (ds.getChildEnhancerList() != null)
dsElem.addContent(writeDatasetScanEnhancer(ds.getChildEnhancerList()));
// @todo Write catalogRefExpander elements
// if ( ds.getCatalogRefExpander() != null )
// dsElem.addContent( writeDatasetScanCatRefExpander( ds.getCatalogRefExpander()));
} else {
if (ds.isValid()) {
dsElem = new Element("catalogRef", defNS);
writeDatasetInfo(ds, dsElem, false, false);
dsElem.setAttribute("href", ds.getXlinkHref(), xlinkNS);
dsElem.setAttribute("title", ds.getName(), xlinkNS);
dsElem.setAttribute("name", "");
dsElem.addContent(writeProperty(new InvProperty("DatasetScan", "true")));
} else {
dsElem = new Element("dataset", defNS);
dsElem.setAttribute("name", "** Misconfigured DatasetScan <" + ds.getPath() + "> **");
dsElem.addContent(new Comment(ds.getInvalidMessage()));
}
}
return dsElem;
}
Element writeDatasetScanFilter(CrawlableDatasetFilter filter) {
Element filterElem = new Element("filter", defNS);
if (filter.getClass().isAssignableFrom(MultiSelectorFilter.class) && filter.getConfigObject() != null) {
for (Object o : ((List) filter.getConfigObject())) {
MultiSelectorFilter.Selector curSelector = (MultiSelectorFilter.Selector) o;
Element curSelectorElem;
if (curSelector.isIncluder())
curSelectorElem = new Element("include", defNS);
else
curSelectorElem = new Element("exclude", defNS);
CrawlableDatasetFilter curFilter = curSelector.getFilter();
if (curFilter instanceof WildcardMatchOnNameFilter) {
curSelectorElem.setAttribute("wildcard", ((WildcardMatchOnNameFilter) curFilter).getWildcardString());
curSelectorElem.setAttribute("atomic", curSelector.isApplyToAtomicDataset() ? "true" : "false");
curSelectorElem.setAttribute("collection", curSelector.isApplyToCollectionDataset() ? "true" : "false");
} else if (curFilter instanceof RegExpMatchOnNameFilter) {
curSelectorElem.setAttribute("regExp", ((RegExpMatchOnNameFilter) curFilter).getRegExpString());
curSelectorElem.setAttribute("atomic", curSelector.isApplyToAtomicDataset() ? "true" : "false");
curSelectorElem.setAttribute("collection", curSelector.isApplyToCollectionDataset() ? "true" : "false");
} else if (curFilter instanceof LastModifiedLimitFilter) {
curSelectorElem.setAttribute("lastModLimitInMillis", Long.toString(((LastModifiedLimitFilter) curFilter).getLastModifiedLimitInMillis()));
curSelectorElem.setAttribute("atomic", curSelector.isApplyToAtomicDataset() ? "true" : "false");
curSelectorElem.setAttribute("collection", curSelector.isApplyToCollectionDataset() ? "true" : "false");
} else
curSelectorElem.addContent(new Comment("Unknown selector type <" + curSelector.getClass().getName() + ">."));
filterElem.addContent(curSelectorElem);
}
} else {
filterElem.addContent(writeDatasetScanUserDefined("crawlableDatasetFilterImpl", filter.getClass().getName(), filter.getConfigObject()));
}
return filterElem;
}
private Element writeDatasetScanNamer(CrawlableDatasetLabeler namer) {
Element namerElem = null;
if (namer != null) {
namerElem = new Element("namer", defNS);
if (namer instanceof MultiLabeler) {
for (CrawlableDatasetLabeler curNamer : ((MultiLabeler) namer).getLabelerList()) {
Element curNamerElem;
if (curNamer instanceof RegExpAndReplaceOnNameLabeler) {
curNamerElem = new Element("regExpOnName", defNS);
curNamerElem.setAttribute("regExp", ((RegExpAndReplaceOnNameLabeler) curNamer).getRegExp());
curNamerElem.setAttribute("replaceString", ((RegExpAndReplaceOnNameLabeler) curNamer).getReplaceString());
namerElem.addContent(curNamerElem);
} else if (curNamer instanceof RegExpAndReplaceOnPathLabeler) {
curNamerElem = new Element("regExpOnPath", defNS);
curNamerElem.setAttribute("regExp", ((RegExpAndReplaceOnPathLabeler) curNamer).getRegExp());
curNamerElem.setAttribute("replaceString", ((RegExpAndReplaceOnPathLabeler) curNamer).getReplaceString());
namerElem.addContent(curNamerElem);
} else {
String tmpMsg = "writeDatasetScanNamer(): unsupported namer <" + curNamer.getClass().getName() + ">.";
logger.warn(tmpMsg);
namerElem.addContent(new Comment(tmpMsg));
}
}
} else {
namerElem.addContent(writeDatasetScanUserDefined("crawlableDatasetLabelerImpl", namer.getClass().getName(), namer.getConfigObject()));
}
}
return namerElem;
}
private Element writeDatasetScanIdentifier(CrawlableDatasetLabeler identifier) {
Element identifierElem = new Element("addID", defNS);
if (identifier != null) {
if (identifier instanceof SimpleLatestProxyDsHandler) {
return identifierElem;
} else {
identifierElem = new Element("addID", defNS);
identifierElem.addContent(writeDatasetScanUserDefined("crawlableDatasetLabelerImpl", identifier.getClass().getName(), identifier.getConfigObject()));
}
}
return identifierElem;
}
private Element writeDatasetScanAddProxies(Map<String, ProxyDatasetHandler> proxyDsHandlers) {
Element addProxiesElem;
// Write addLatest element if only proxyDsHandler and named "latest.xml".
if (proxyDsHandlers.size() == 1 && proxyDsHandlers.containsKey("latest.xml")) {
Object o = proxyDsHandlers.get("latest.xml");
if (o instanceof SimpleLatestProxyDsHandler) {
SimpleLatestProxyDsHandler pdh = (SimpleLatestProxyDsHandler) o;
String name = pdh.getProxyDatasetName();
boolean top = pdh.isLocateAtTopOrBottom();
String serviceName = pdh.getProxyDatasetService(null).getName();
addProxiesElem = new Element("addLatest", defNS);
if (name.equals("latest.xml") && top && serviceName.equals("latest"))
return addProxiesElem;
else {
Element simpleLatestElem = new Element("simpleLatest", defNS);
simpleLatestElem.setAttribute("name", name);
simpleLatestElem.setAttribute("top", top ? "true" : "false");
simpleLatestElem.setAttribute("servicName", serviceName);
addProxiesElem.addContent(simpleLatestElem);
return addProxiesElem;
}
}
}
// Write "addProxies" element
addProxiesElem = new Element("addProxies", defNS);
for (Map.Entry<String, ProxyDatasetHandler> entry : proxyDsHandlers.entrySet()) {
String curName = entry.getKey();
ProxyDatasetHandler curPdh = entry.getValue();
if (curPdh instanceof SimpleLatestProxyDsHandler) {
SimpleLatestProxyDsHandler sPdh = (SimpleLatestProxyDsHandler) curPdh;
Element simpleLatestElem = new Element("simpleLatest", defNS);
simpleLatestElem.setAttribute("name", sPdh.getProxyDatasetName());
simpleLatestElem.setAttribute("top", sPdh.isLocateAtTopOrBottom() ? "true" : "false");
simpleLatestElem.setAttribute("servicName", sPdh.getProxyDatasetService(null).getName());
addProxiesElem.addContent(simpleLatestElem);
} else if (curPdh instanceof LatestCompleteProxyDsHandler) {
LatestCompleteProxyDsHandler lcPdh = (LatestCompleteProxyDsHandler) curPdh;
Element latestElem = new Element("latestComplete", defNS);
latestElem.setAttribute("name", lcPdh.getProxyDatasetName());
latestElem.setAttribute("top", lcPdh.isLocateAtTopOrBottom() ? "true" : "false");
latestElem.setAttribute("servicName", lcPdh.getProxyDatasetService(null).getName());
latestElem.setAttribute("lastModifiedLimit", Long.toString(lcPdh.getLastModifiedLimit()));
addProxiesElem.addContent(latestElem);
} else {
logger.warn("writeDatasetScanAddProxies(): unknown type of ProxyDatasetHandler <" + curPdh.getProxyDatasetName() + ">.");
// latestAdderElem.addContent( writeDatasetScanUserDefined( "datasetInserterImpl", latestAdder.getClass().getName(), latestAdder.getConfigObject() ) );
}
}
return addProxiesElem;
}
private Element writeDatasetScanSorter(CrawlableDatasetSorter sorter) {
Element sorterElem = new Element("sort", defNS);
if (sorter instanceof LexigraphicByNameSorter) {
Element lexElem = new Element("lexigraphicByName", defNS);
lexElem.setAttribute("increasing", ((LexigraphicByNameSorter) sorter).isIncreasing() ? "true" : "false");
sorterElem.addContent(lexElem);
} else {
sorterElem.addContent(writeDatasetScanUserDefined("crawlableDatasetSorterImpl", sorter.getClass().getName(), sorter.getConfigObject()));
}
return sorterElem;
}
private List<Element> writeDatasetScanEnhancer(List<DatasetEnhancer> enhancerList) {
List<Element> enhancerElemList = new ArrayList<>();
int timeCovCount = 0;
for (DatasetEnhancer curEnhancer : enhancerList) {
if (curEnhancer instanceof RegExpAndDurationTimeCoverageEnhancer) {
if (timeCovCount > 0) {
logger.warn("writeDatasetScanEnhancer(): More than one addTimeCoverage element, skipping.");
continue;
}
timeCovCount++;
Element timeCovElem = new Element("addTimeCoverage", defNS);
RegExpAndDurationTimeCoverageEnhancer timeCovEnhancer = (RegExpAndDurationTimeCoverageEnhancer) curEnhancer;
timeCovElem.setAttribute("datasetNameMatchPattern", timeCovEnhancer.getMatchPattern());
timeCovElem.setAttribute("startTimeSubstitutionPattern", timeCovEnhancer.getSubstitutionPattern());
timeCovElem.setAttribute("duration", timeCovEnhancer.getDuration());
enhancerElemList.add(timeCovElem);
} else {
enhancerElemList.add(writeDatasetScanUserDefined("datasetEnhancerImpl", curEnhancer.getClass().getName(), curEnhancer.getConfigObject()));
}
}
return enhancerElemList;
}
private Element writeDatasetScanUserDefined(String userDefName, String className, Object configObj) {
Element userDefElem = new Element(userDefName, defNS);
userDefElem.setAttribute("className", className);
if (configObj != null) {
if (configObj instanceof Element)
userDefElem.addContent((Element) configObj);
else
userDefElem.addContent(new Comment("This class <" + className + "> not yet supported. This XML is missing configuration information (of type " + configObj.getClass().getName() + ")."));
}
return userDefElem;
}
private void writeDatasetInfo(InvDatasetImpl ds, Element dsElem, boolean doNestedDatasets, boolean showNcML) {
dsElem.setAttribute("name", ds.getName());
// other attributes, note the others get made into an element
if ((ds.getCollectionType() != null) && (ds.getCollectionType() != CollectionType.NONE))
dsElem.setAttribute("collectionType", ds.getCollectionType().toString());
if (ds.isHarvest())
dsElem.setAttribute("harvest", "true");
if (ds.getID() != null)
dsElem.setAttribute("ID", ds.getID());
if (ds.getUrlPath() != null)
dsElem.setAttribute("urlPath", ds.getUrlPath());
if (ds.getRestrictAccess() != null)
dsElem.setAttribute("restrictAccess", ds.getRestrictAccess());
// services (local only)
for (InvService service : ds.getServicesLocal()) {
dsElem.addContent(writeService(service));
}
// thredds metadata
writeThreddsMetadata(dsElem, ds.getLocalMetadata());
writeInheritedMetadata(dsElem, ds.getLocalMetadataInheritable());
// writeInheritedMetadata( dsElem, ds.getCat6Metadata()); // LOOK can we get rid of this?
// access (local only)
for (InvAccess a : ds.getAccessLocal()) {
dsElem.addContent(writeAccess( (InvAccessImpl) a));
}
if (showNcML && ds.getNcmlElement() != null) {
org.jdom2.Element ncml = ds.getNcmlElement().clone();
ncml.detach();
dsElem.addContent(ncml);
}
if (!doNestedDatasets) return;
// nested datasets
for (InvDataset nested : ds.getDatasets()) {
if (nested instanceof InvDatasetScan)
dsElem.addContent(writeDatasetScan((InvDatasetScan) nested));
else if (nested instanceof InvCatalogRef)
dsElem.addContent(writeCatalogRef((InvCatalogRef) nested));
else
dsElem.addContent(writeDataset( (InvDatasetImpl) nested));
}
}
protected Element writeDate(String name, DateType date) {
Element dateElem = new Element(name, defNS);
dateElem.addContent(date.getText());
if (date.getType() != null)
dateElem.setAttribute("type", date.getType());
if (date.getFormat() != null)
dateElem.setAttribute("format", date.getFormat());
return dateElem;
}
private Element writeDocumentation(InvDocumentation doc, String name) {
Element docElem = new Element(name, defNS);
if (doc.getType() != null)
docElem.setAttribute("type", doc.getType());
if (doc.hasXlink()) {
docElem.setAttribute("href", doc.getXlinkHref(), xlinkNS);
if (!doc.getXlinkTitle().equals(doc.getURI().toString()))
docElem.setAttribute("title", doc.getXlinkTitle(), xlinkNS);
}
String inline = doc.getInlineContent();
if (inline != null)
docElem.addContent(inline);
return docElem;
}
public Element writeGeospatialCoverage(ThreddsMetadata.GeospatialCoverage gc) {
Element elem = new Element("geospatialCoverage", defNS);
if (gc.getZPositive().equals("down"))
elem.setAttribute("zpositive", gc.getZPositive());
if (gc.getNorthSouthRange() != null)
writeGeospatialRange(elem, new Element("northsouth", defNS), gc.getNorthSouthRange());
if (gc.getEastWestRange() != null)
writeGeospatialRange(elem, new Element("eastwest", defNS), gc.getEastWestRange());
if (gc.getUpDownRange() != null)
writeGeospatialRange(elem, new Element("updown", defNS), gc.getUpDownRange());
// serialize isGlobal
java.util.List<ThreddsMetadata.Vocab> names = gc.getNames();
ThreddsMetadata.Vocab global = new ThreddsMetadata.Vocab("global", null);
if (gc.isGlobal() && !names.contains(global)) {
names.add(global);
} else if (!gc.isGlobal() && names.contains(global)) {
names.remove(global);
}
for (ThreddsMetadata.Vocab name : names) {
elem.addContent(writeControlledVocabulary(name, "name"));
}
return elem;
}
private void writeGeospatialRange(Element parent, Element elem, ThreddsMetadata.Range r) {
if (r == null) return;
elem.addContent(new Element("start", defNS).setText(Double.toString(r.getStart())));
elem.addContent(new Element("size", defNS).setText(Double.toString(r.getSize())));
if (r.hasResolution())
elem.addContent(new Element("resolution", defNS).setText(Double.toString(r.getResolution())));
if (r.getUnits() != null)
elem.addContent(new Element("units", defNS).setText(r.getUnits()));
parent.addContent(elem);
}
private Element writeMetadata(InvMetadata mdata) {
Element mdataElem = new Element("metadata", defNS);
if (mdata.getMetadataType() != null)
mdataElem.setAttribute("metadataType", mdata.getMetadataType());
if (mdata.isInherited())
mdataElem.setAttribute("inherited", "true");
String ns = mdata.getNamespaceURI();
if ((ns != null) && !ns.equals(XMLEntityResolver.CATALOG_NAMESPACE_10)) {
Namespace mdataNS = Namespace.getNamespace(mdata.getNamespacePrefix(), ns);
mdataElem.addNamespaceDeclaration(mdataNS);
}
if (mdata.hasXlink()) {
mdataElem.setAttribute("href", mdata.getXlinkHref(), xlinkNS);
if (mdata.getXlinkTitle() != null)
mdataElem.setAttribute("title", mdata.getXlinkTitle(), xlinkNS);
} else if (mdata.getThreddsMetadata() != null) {
writeThreddsMetadata(mdataElem, mdata.getThreddsMetadata());
} else {
// inline non-thredds case
MetadataConverterIF converter = mdata.getConverter();
if ((converter != null) && mdata.getContentObject() != null) {
if (mdata.getContentObject() instanceof Element) { // special case
Element mdataOrg = (Element) mdata.getContentObject();
List<Element> children = mdataOrg.getChildren();
for (Element child : children) {
mdataElem.addContent( child.clone());
}
} else {
//org.w3c.dom.Element dome = toDOM(mdataElem);
converter.addMetadataContent(mdataElem, mdata.getContentObject());
//mdataElem = toJDOM(dome);
mdataElem.detach();
}
}
}
return mdataElem;
}
private Element writeProperty(InvProperty prop) {
Element propElem = new Element("property", defNS);
propElem.setAttribute("name", prop.getName());
propElem.setAttribute("value", prop.getValue());
return propElem;
}
protected Element writeSource(String elementName, ThreddsMetadata.Source p) {
Element elem = new Element(elementName, defNS);
elem.addContent(writeControlledVocabulary(p.getNameVocab(), "name"));
Element contact = new Element("contact", defNS);
if (p.getUrl() != null)
contact.setAttribute("url", p.getUrl());
if (p.getEmail() != null)
contact.setAttribute("email", p.getEmail());
elem.addContent(contact);
return elem;
}
private Element writeService(InvService service) {
Element serviceElem = new Element("service", defNS);
serviceElem.setAttribute("name", service.getName());
serviceElem.setAttribute("serviceType", service.getServiceType().toString());
serviceElem.setAttribute("base", service.getBase());
if ((service.getSuffix() != null) && (service.getSuffix().length() > 0))
serviceElem.setAttribute("suffix", service.getSuffix());
// properties
for (InvProperty p : service.getProperties()) {
serviceElem.addContent(writeProperty(p));
}
// services
for (InvService nested : service.getServices()) {
serviceElem.addContent(writeService(nested));
}
// dataset roots
if (raw) {
for (InvProperty p : service.getDatasetRoots()) {
serviceElem.addContent(writeDatasetRoot(p));
}
}
return serviceElem;
}
private Element writeDataSize(double size) {
Element sizeElem = new Element("dataSize", defNS);
// want exactly the number of bytes
if (useBytesForDataSize) {
sizeElem.setAttribute("units", "bytes");
long bytes = (long) size;
sizeElem.setText(Long.toString(bytes));
return sizeElem;
}
// otherwise choose appropriate unit
String unit;
if (size > 1.0e15) {
unit = "Pbytes";
size *= 1.0e-15;
} else if (size > 1.0e12) {
unit = "Tbytes";
size *= 1.0e-12;
} else if (size > 1.0e9) {
unit = "Gbytes";
size *= 1.0e-9;
} else if (size > 1.0e6) {
unit = "Mbytes";
size *= 1.0e-6;
} else if (size > 1.0e3) {
unit = "Kbytes";
size *= 1.0e-3;
} else {
unit = "bytes";
}
sizeElem.setAttribute("units", unit);
sizeElem.setText(ucar.unidata.util.Format.d(size, 4));
return sizeElem;
}
/* protected void writeCat6InheritedMetadata( Element elem, ThreddsMetadata tmi) {
if ((tmi.getDataType() == null) && (tmi.getServiceName() == null) &&
(tmi.getAuthority() == null) && ( tmi.getProperties().size() == 0))
return;
Element mdataElem = new Element("metadata", defNS);
mdataElem.setAttribute("inherited", "true");
writeThreddsMetadata( mdataElem, tmi);
elem.addContent( mdataElem);
} */
protected void writeInheritedMetadata(Element elem, ThreddsMetadata tmi) {
Element mdataElem = new Element("metadata", defNS);
mdataElem.setAttribute("inherited", "true");
writeThreddsMetadata(mdataElem, tmi);
if (mdataElem.getChildren().size() > 0)
elem.addContent(mdataElem);
}
protected void writeThreddsMetadata(Element elem, ThreddsMetadata tmg) {
if (tmg.getServiceName() != null) {
Element serviceNameElem = new Element("serviceName", defNS);
serviceNameElem.setText(tmg.getServiceName());
elem.addContent(serviceNameElem);
}
if (tmg.getAuthority() != null) {
Element authElem = new Element("authority", defNS);
authElem.setText(tmg.getAuthority());
elem.addContent(authElem);
}
if ((tmg.getDataType() != null) && (tmg.getDataType() != FeatureType.NONE) && (tmg.getDataType() != FeatureType.ANY)) {
Element dataTypeElem = new Element("dataType", defNS);
dataTypeElem.setText(tmg.getDataType().toString());
elem.addContent(dataTypeElem);
}
if ((tmg.getDataFormatType() != null) && (tmg.getDataFormatType() != DataFormatType.NONE)) {
Element dataFormatElem = new Element("dataFormat", defNS);
dataFormatElem.setText(tmg.getDataFormatType().toString());
elem.addContent(dataFormatElem);
}
if (tmg.hasDataSize())
elem.addContent(writeDataSize(tmg.getDataSize()));
List<InvDocumentation> docList = tmg.getDocumentation();
for (InvDocumentation doc : docList) {
elem.addContent(writeDocumentation(doc, "documentation"));
}
List<ThreddsMetadata.Contributor> contribList = tmg.getContributors();
for (ThreddsMetadata.Contributor c : contribList) {
elem.addContent(writeContributor(c));
}
List<ThreddsMetadata.Source> creatorList = tmg.getCreators();
for (ThreddsMetadata.Source p : creatorList) {
elem.addContent(writeSource("creator", p));
}
List<ThreddsMetadata.Vocab> kewordList = tmg.getKeywords();
for (ThreddsMetadata.Vocab v : kewordList) {
elem.addContent(writeControlledVocabulary(v, "keyword"));
}
List<InvMetadata> mdList = tmg.getMetadata();
for (InvMetadata m : mdList) {
elem.addContent(writeMetadata(m));
}
List<ThreddsMetadata.Vocab> projList = tmg.getProjects();
for (ThreddsMetadata.Vocab v : projList) {
elem.addContent(writeControlledVocabulary(v, "project"));
}
List<InvProperty> propertyList = tmg.getProperties();
for (InvProperty p : propertyList) {
elem.addContent(writeProperty(p));
}
List<ThreddsMetadata.Source> pubList = tmg.getPublishers();
for (ThreddsMetadata.Source p : pubList) {
elem.addContent(writeSource("publisher", p));
}
List<DateType> dateList = tmg.getDates();
for (DateType d : dateList) {
elem.addContent(writeDate("date", d));
}
ThreddsMetadata.GeospatialCoverage gc = tmg.getGeospatialCoverage();
if ((gc != null) && !gc.isEmpty())
elem.addContent(writeGeospatialCoverage(gc));
DateRange tc = tmg.getTimeCoverage();
if (tc != null)
elem.addContent(writeTimeCoverage(tc));
List<ThreddsMetadata.Variables> varList = tmg.getVariables();
for (ThreddsMetadata.Variables v : varList) {
elem.addContent(writeVariables(v));
}
String varMapLink = tmg.getVariableMap();
if (varMapLink != null) {
Element velem = new Element("variableMap", defNS);
velem.setAttribute("href", varMapLink, xlinkNS);
velem.setAttribute("title", "variables", xlinkNS);
elem.addContent(velem);
}
}
protected Element writeTimeCoverage(DateRange t) {
Element elem = new Element("timeCoverage", defNS);
DateType start = t.getStart();
DateType end = t.getEnd();
TimeDuration duration = t.getDuration();
TimeDuration resolution = t.getResolution();
if (t.useStart() && (start != null) && !start.isBlank()) {
Element startElem = new Element("start", defNS);
startElem.setText(start.toString());
elem.addContent(startElem);
}
if (t.useEnd() && (end != null) && !end.isBlank()) {
Element telem = new Element("end", defNS);
telem.setText(end.toString());
elem.addContent(telem);
}
if (t.useDuration() && (duration != null) && !duration.isBlank()) {
Element telem = new Element("duration", defNS);
telem.setText(duration.toString());
elem.addContent(telem);
}
if (t.useResolution() && (resolution != null) && !resolution.isBlank()) {
Element telem = new Element("resolution", defNS);
telem.setText(t.getResolution().toString());
elem.addContent(telem);
}
return elem;
}
protected Element writeVariable(ThreddsMetadata.Variable v) {
Element elem = new Element("variable", defNS);
if (v.getName() != null)
elem.setAttribute("name", v.getName());
if (v.getDescription() != null) {
String desc = v.getDescription().trim();
if (desc.length() > 0)
elem.setText(v.getDescription());
}
if (v.getVocabularyName() != null)
elem.setAttribute("vocabulary_name", v.getVocabularyName());
if (v.getUnits() != null)
elem.setAttribute("units", v.getUnits());
String id = v.getVocabularyId();
if (id != null)
elem.setAttribute("vocabulary_id", id);
return elem;
}
protected Element writeVariables(ThreddsMetadata.Variables vs) {
Element elem = new Element("variables", defNS);
if (vs.getVocabulary() != null)
elem.setAttribute("vocabulary", vs.getVocabulary());
if (vs.getVocabHref() != null)
elem.setAttribute("href", vs.getVocabHref(), xlinkNS);
if (vs.getMapHref() != null) { // variable map
Element mapElem = new Element("variableMap", defNS);
mapElem.setAttribute("href", vs.getMapHref(), xlinkNS);
elem.addContent(mapElem);
} else { // inline variables
List<ThreddsMetadata.Variable> varList = vs.getVariableList();
for (ThreddsMetadata.Variable v : varList) {
elem.addContent(writeVariable(v));
}
}
return elem;
}
/* public org.w3c.dom.Element toDOM( Element elem) {
try {
if (domOut == null) domOut = new DOMOutputter();
return domOut.output(elem);
} catch (JDOMException e) {
System.out.println("InvCatalogFactory6.readMetadata.toDom error " + e);
return null;
}
}
public Element toJDOM( org.w3c.dom.Element domElement) {
return builder.build(domElement);
} */
}