/******************************************************************************* * Copyright 2012 University of Southern California * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * This code was developed by the Information Integration Group as part * of the Karma project at the Information Sciences Institute of the * University of Southern California. For more information, publications, * and related projects, please see: http://www.isi.edu/integration ******************************************************************************/ package edu.isi.karma.model.serialization; import java.io.File; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import org.apache.log4j.Logger; import com.hp.hpl.jena.query.Query; import com.hp.hpl.jena.query.QueryExecution; import com.hp.hpl.jena.query.QueryExecutionFactory; import com.hp.hpl.jena.query.QueryFactory; import com.hp.hpl.jena.query.QuerySolution; import com.hp.hpl.jena.query.ResultSet; import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.NodeIterator; import com.hp.hpl.jena.rdf.model.Property; import com.hp.hpl.jena.rdf.model.RDFNode; import com.hp.hpl.jena.rdf.model.Resource; import com.hp.hpl.jena.rdf.model.ResourceFactory; import edu.isi.karma.modeling.Namespaces; import edu.isi.karma.modeling.Prefixes; import edu.isi.karma.rep.alignment.Label; import edu.isi.karma.rep.model.Argument; import edu.isi.karma.rep.model.ArgumentType; import edu.isi.karma.rep.model.Atom; import edu.isi.karma.rep.model.ClassAtom; import edu.isi.karma.rep.model.IndividualPropertyAtom; import edu.isi.karma.rep.sources.Attribute; import edu.isi.karma.rep.sources.AttributeRequirement; import edu.isi.karma.rep.sources.DataSource; import edu.isi.karma.rep.sources.IOType; import edu.isi.karma.rep.sources.Source; public class DataSourceLoader extends SourceLoader { static Logger logger = Logger.getLogger(DataSourceLoader.class); private static DataSourceLoader instance = null; private static final int DEFAULT_SOURCE_RESULTS_SIZE = 10; protected DataSourceLoader() { // Exists only to defeat instantiation. } public static DataSourceLoader getInstance() { if (instance == null) { instance = new DataSourceLoader(); } return instance; } @Override public DataSource getSourceByUri(String uri) { Model m = Repository.Instance().getNamedModel(uri); if (m == null) return null; DataSource source = importSourceFromJenaModel(m); return source; } @Override public void deleteSourceByUri(String uri) { Repository.Instance().clearNamedModel(uri); String source_id = uri.substring(uri.lastIndexOf("/") + 1, uri.length() - 1); String dir = Repository.Instance().SOURCE_REPOSITORY_DIR; String fileName = source_id + Repository.Instance().getFileExtension(Repository.Instance().LANG); File f = new File(dir + fileName); try { if (f.exists()) { if (!f.delete()) logger.debug("The file " + fileName + " cannot be deleted from " + dir); else logger.debug("The file " + fileName + " has been deleted from " + dir); } else logger.debug("The file " + fileName + " does not exist in " + dir); } catch (Throwable t) { logger.debug("cannot delete the file " + fileName + " from " + dir + " because " + t.getMessage()); } } /** * returns the source id, name, address of all sources in the repository * @param limit : maximum number of results, null value means all the sources * @return */ @Override public List<Source> getSourcesAbstractInfo(Integer sourceLimit) { List<Source> sourceList = new ArrayList<Source>(); Model model = Repository.Instance().getModel(); String source_id = ""; String source_name = ""; // Create a new query String queryString = "PREFIX " + Prefixes.KARMA + ": <" + Namespaces.KARMA + "> \n" + "SELECT ?s ?name \n" + "WHERE { \n" + " ?s a " + Prefixes.KARMA + ":Source . \n" + " OPTIONAL {?s " + Prefixes.KARMA + ":hasName ?name .} \n" + " } \n"; if (sourceLimit != null) { if (sourceLimit.intValue() < 0) sourceLimit = DEFAULT_SOURCE_RESULTS_SIZE; queryString += "LIMIT " + String.valueOf(sourceLimit.intValue() + "\n"); } logger.debug("query= \n" + queryString); Query query = QueryFactory.create(queryString); // Execute the query and obtain results QueryExecution qexec = QueryExecutionFactory.create(query, model); try { ResultSet results = qexec.execSelect() ; if (!results.hasNext()) logger.info("query does not return any answer."); // ResultSetFormatter.out(System.out, results, query) ; for ( ; results.hasNext() ; ) { QuerySolution soln = results.nextSolution() ; RDFNode s = soln.get("s") ; // Get a result variable by name. RDFNode name = soln.get("name") ; // Get a result variable by name. if (s == null) { logger.info("source id is null."); continue; } String source_uri = s.toString(); source_id = source_uri.substring(source_uri.lastIndexOf("/") + 1, source_uri.length() - 1); logger.debug("source uri: " + source_uri); logger.debug("source id: " + source_id); if (name != null && name.isLiteral()) source_name = name.asLiteral().getString(); logger.debug("source name: " + source_name); if (source_id.trim().length() > 0) sourceList.add(new DataSource(source_id, source_name)); else logger.info("length of source id is zero."); } return sourceList; } catch (Exception e) { logger.info(e.getMessage()); return null; } finally { qexec.close() ; } } /** * returns all the sources in the repository along with their complete information * probably this method is not useful since fetching all the sources with their complete * information takes long time. * @return */ @Override public List<Source> getSourcesDetailedInfo(Integer sourceLimit) { List<Source> sourceList = getSourcesAbstractInfo(sourceLimit); List<Source> sourceListCompleteInfo = new ArrayList<Source>(); for (Source s : sourceList) { sourceListCompleteInfo.add(getSourceByUri(s.getUri())); } return sourceListCompleteInfo; } /** * Searches the repository to find the sources whose model matches the semantic model parameter. * @param semanticModel The input model whose pattern will be searched in the repository * @param ioType declares which one of the source input or source output will be tested for matching * @param sourceLimit maximum number of sources that will be fetched * @return a hashmap of all found sources and a mapping from the found source parameters to the model parameters. * This help us later to how to join the model's corresponding source and the matched source */ public Map<DataSource, Map<String, String>> getDataSourcesByIOPattern(edu.isi.karma.rep.model.Model semanticModel, Integer sourceLimit) { if (semanticModel == null || semanticModel.getAtoms() == null || semanticModel.getAtoms().size() == 0) { logger.info("The input model is nul or it does not have any atom"); return null; } Map<DataSource, Map<String, String>> sourcesAndMappings = new HashMap<DataSource, Map<String,String>>(); Map<String, Map<String, String>> sourceIdsAndMappings = semanticModel.findInJenaModel(Repository.Instance().getModel(), sourceLimit); if (sourceIdsAndMappings == null) return null; for (String sourceId : sourceIdsAndMappings.keySet()) { Model m = Repository.Instance().getNamedModel(sourceId); if (m != null) sourcesAndMappings.put(importSourceFromJenaModel(m), sourceIdsAndMappings.get(sourceId)); } return sourcesAndMappings; } /** * Searches the repository to find the sources whose model is contained in the semantic model parameter. * Note that the services in the return list only include the operations that match the model parameter. * @param semanticModel The input model whose pattern will be searched in the repository * @param ioType declares which one of the service input or service output will be tested for matching. * @param operationsLimit maximum number of operations that will be fetched * @return a hashmap of all found sources and a mapping from the found source parameters to the model parameters. * This help us later to how to join the model's corresponding source and the matched service */ public Map<DataSource, Map<String, String>> getDataSourcesContainedInModel(edu.isi.karma.rep.model.Model semanticModel, Integer sourceLimit) { List<Source> sourceList = getSourcesDetailedInfo(sourceLimit); Map<DataSource, Map<String, String>> sourcesAndMappings = new HashMap<DataSource, Map<String,String>>(); Model jenaModel = semanticModel.getJenaModel(); for (Source source : sourceList) { if (!(source instanceof DataSource)) continue; edu.isi.karma.rep.model.Model m = ((DataSource)source).getModel(); if (m == null) continue; Map<String, Map<String, String>> sourceIdsAndMappings = m.findInJenaModel(jenaModel, null); if (sourceIdsAndMappings == null) continue; Iterator<String> itr = sourceIdsAndMappings.keySet().iterator(); if (itr.hasNext()) { String key = itr.next(); sourcesAndMappings.put((DataSource)source, sourceIdsAndMappings.get(key)); } } return sourcesAndMappings; } /** * From the source model, returns the source object * If the list of operation ids is null, it includes all the operations, otherwise it only * fetches the mentioned operations * If operationLimit=null and operationIds=null returns all operations * If operationLimit=null, it returns the all specified operations * If operationIds=null, it returns the first (opLimit) number of operations * If both of them are not null, it returns the first (opLimit) number of specified operations * @param model * @param operationIds * @return */ @Override public DataSource importSourceFromJenaModel(Model model) { logger.debug("model size: " + model.getGraph().size()); String source_name = ""; String source_uri = ""; String source_id = ""; // source id source_uri = model.getNsPrefixURI(""); logger.debug("source uri: " + source_uri); // source local id source_id = source_uri.substring(source_uri.lastIndexOf("/") + 1, source_uri.length() - 1); logger.debug("source id: " + source_id); Property has_name_property = model.getProperty(Namespaces.KARMA + "hasName"); Resource source_resource = model.getResource(source_uri); NodeIterator nodeIterator = null; RDFNode node = null; // source name nodeIterator = model.listObjectsOfProperty(source_resource, has_name_property); if (nodeIterator.hasNext() && (node = nodeIterator.next()).isLiteral()) { source_name = node.asLiteral().getString(); logger.debug("source name: " + source_name); } else logger.debug("source does not have a name."); DataSource source = new DataSource(source_id); source.setName(source_name); source.setVariables(getVariables(model, source_resource)); source.setAttributes(getAttributes(model, source_resource)); source.setModel(getSemanticModel(model, source_resource)); return source; } private List<String> getVariables(Model model, Resource source_resource) { Property has_variable_property = model.getProperty(Namespaces.KARMA + "hasVariable"); List<String> variables = new ArrayList<String>(); NodeIterator nodeIterator = null; RDFNode node = null; // hasAttribute nodeIterator = model.listObjectsOfProperty(source_resource, has_variable_property); while ( nodeIterator.hasNext()) { node = nodeIterator.next(); if (!node.isResource()) { logger.info("object of the hasAttribute property is not a resource."); continue; } variables.add(node.asResource().getLocalName()); } return variables; } private List<Attribute> getAttributes(Model model, Resource source_resource) { Property has_attribute_property = model.getProperty(Namespaces.KARMA + "hasAttribute"); List<Attribute> attList = new ArrayList<Attribute>(); NodeIterator nodeIterator = null; RDFNode node = null; // hasAttribute nodeIterator = model.listObjectsOfProperty(source_resource, has_attribute_property); while ( nodeIterator.hasNext()) { node = nodeIterator.next(); if (!node.isResource()) { logger.info("object of the hasAttribute property is not a resource."); continue; } attList.add(getSingleAttribute(model, node.asResource())); } return attList; } private Attribute getSingleAttribute(Model model, Resource att_resource) { String att_id = ""; String att_name = ""; Property has_name_property = model.getProperty(Namespaces.KARMA + "hasName"); // attribute id att_id = att_resource.getLocalName(); logger.debug("attribute id: " + att_id); NodeIterator nodeIterator = null; RDFNode node = null; // attribute name nodeIterator = model.listObjectsOfProperty(att_resource, has_name_property); if (nodeIterator.hasNext() && (node = nodeIterator.next()).isLiteral()) { att_name = node.asLiteral().getString(); logger.debug("attribute name: " + att_name); } else logger.debug("attribute does not have a name."); Attribute att = new Attribute(att_id, att_resource.getNameSpace(), att_name, IOType.NONE, AttributeRequirement.NONE); return att; } private edu.isi.karma.rep.model.Model getSemanticModel(Model model, Resource source_resource) { Property has_model_property = model.getProperty(Namespaces.KARMA + "hasModel"); Property has_atom_property = model.getProperty(Namespaces.KARMA + "hasAtom"); NodeIterator nodeIterator = null; RDFNode modelNode = null; RDFNode atomNode = null; // hasModel nodeIterator = model.listObjectsOfProperty(source_resource, has_model_property); if (!nodeIterator.hasNext() || !(modelNode = nodeIterator.next()).isResource()) { logger.info("There is no model resource."); return null; } edu.isi.karma.rep.model.Model semanticModel = new edu.isi.karma.rep.model.Model(modelNode.asResource().getLocalName()); List<Atom> atoms = new ArrayList<Atom>(); // hasAtom nodeIterator = model.listObjectsOfProperty(modelNode.asResource(), has_atom_property); while ( nodeIterator.hasNext()) { atomNode = nodeIterator.next(); if (!atomNode.isResource()) { logger.info("object of the hasAtom property is not a resource."); continue; } atoms.add(getAtom(model, atomNode.asResource())); } semanticModel.setAtoms(atoms); return semanticModel; } private Atom getAtom(Model model, Resource atom_resource) { Property rdf_type = model.getProperty(Namespaces.RDF + "type"); NodeIterator nodeIterator = null; RDFNode node = null; String classAtomUri = Namespaces.SWRL + "ClassAtom"; String propertyAtomUri = Namespaces.SWRL + "IndividualPropertyAtom"; // atom type nodeIterator = model.listObjectsOfProperty(atom_resource, rdf_type); if (!nodeIterator.hasNext() || !(node = nodeIterator.next()).isResource()) { logger.info("The atom type is not specified."); return null; } if (node.asResource().getURI().equalsIgnoreCase(classAtomUri)) { logger.debug("The atom is a ClassAtom"); return getClassAtom(model, atom_resource); } else if (node.asResource().getURI().equalsIgnoreCase(propertyAtomUri)) { logger.debug("The atom is an IndividualPropertyAtom"); return getPropertyAtom(model, atom_resource); } return null; } private ClassAtom getClassAtom(Model model, Resource atom_resource) { String predicateUri = null; String predicatePrefix = null; String predicateNs = null; String argument1Id = null; String argument1Type = null; Resource attribute = ResourceFactory.createResource(Namespaces.KARMA + "Attribute"); Resource variable = ResourceFactory.createResource(Namespaces.SWRL + "Variable"); Property class_predicate_property = model.getProperty(Namespaces.SWRL + "classPredicate"); Property argument1_property = model.getProperty(Namespaces.SWRL + "argument1"); NodeIterator nodeIterator = null; RDFNode node = null; // atom class predicate nodeIterator = model.listObjectsOfProperty(atom_resource, class_predicate_property); if (!nodeIterator.hasNext() || !(node = nodeIterator.next()).isResource()) { logger.info("The class predicate resource is not specified."); return null; } predicateUri = node.asResource().getURI(); logger.debug("The atom predicate is: " + predicateUri); predicateNs = node.asResource().getNameSpace(); predicatePrefix = model.getNsURIPrefix(predicateNs); // atom argument1 nodeIterator = model.listObjectsOfProperty(atom_resource, argument1_property); if (nodeIterator.hasNext() && (node = nodeIterator.next()).isResource()) { argument1Id = node.asResource().getLocalName(); logger.debug("The atom argument1 is: " + argument1Id); if (isInstanceOfTheClass(node.asResource(), attribute)) argument1Type = ArgumentType.ATTRIBUTE; else if (isInstanceOfTheClass(node.asResource(), variable)) argument1Type = ArgumentType.VARIABLE; } else { logger.info("atom does not have an argument1."); return null; } Label predicateName = new Label(predicateUri, predicateNs, predicatePrefix); Argument arg1 = new Argument(argument1Id, argument1Id, argument1Type); ClassAtom classAtom = new ClassAtom(predicateName, arg1); return classAtom; } private IndividualPropertyAtom getPropertyAtom(Model model, Resource atom_resource) { String predicateUri = null; String predicatePrefix = null; String predicateNs = null; String argument1Id = null; String argument2Id = null; String argument1Type = null; String argument2Type = null; Resource attribute = ResourceFactory.createResource(Namespaces.KARMA + "Attribute"); Resource variable = ResourceFactory.createResource(Namespaces.SWRL + "Variable"); Property property_predicate_property = model.getProperty(Namespaces.SWRL + "propertyPredicate"); Property argument1_property = model.getProperty(Namespaces.SWRL + "argument1"); Property argument2_property = model.getProperty(Namespaces.SWRL + "argument2"); NodeIterator nodeIterator = null; RDFNode node = null; // atom class predicate nodeIterator = model.listObjectsOfProperty(atom_resource, property_predicate_property); if (!nodeIterator.hasNext() || !(node = nodeIterator.next()).isResource()) { logger.info("The property predicate resource is not specified."); return null; } predicateUri = node.asResource().getURI(); logger.debug("The atom predicate is: " + predicateUri); predicateNs = node.asResource().getNameSpace(); predicatePrefix = model.getNsURIPrefix(predicateNs); // atom argument1 nodeIterator = model.listObjectsOfProperty(atom_resource, argument1_property); if (nodeIterator.hasNext() && (node = nodeIterator.next()).isResource()) { argument1Id = node.asResource().getLocalName(); logger.debug("The atom argument1 is: " + argument1Id); if (isInstanceOfTheClass(node.asResource(), attribute)) argument1Type = ArgumentType.ATTRIBUTE; else if (isInstanceOfTheClass(node.asResource(), variable)) argument1Type = ArgumentType.VARIABLE; } else { logger.info("atom does not have an argument1."); return null; } // atom argument2 nodeIterator = model.listObjectsOfProperty(atom_resource, argument2_property); if (nodeIterator.hasNext() && (node = nodeIterator.next()).isResource()) { argument2Id = node.asResource().getLocalName(); logger.debug("The atom argument2 is: " + argument2Id); if (isInstanceOfTheClass(node.asResource(), attribute)) argument2Type = ArgumentType.ATTRIBUTE; else if (isInstanceOfTheClass(node.asResource(), variable)) argument2Type = ArgumentType.VARIABLE; } else { logger.info("atom does not have an argument2."); return null; } Label predicateName = new Label(predicateUri, predicateNs, predicatePrefix); Argument arg1 = new Argument(argument1Id, argument1Id, argument1Type); Argument arg2 = new Argument(argument2Id, argument2Id, argument2Type); IndividualPropertyAtom propertyAtom = new IndividualPropertyAtom(predicateName, arg1, arg2); return propertyAtom; } private boolean isInstanceOfTheClass(Resource resource, Resource class_resource) { Property type_property = ResourceFactory.createProperty(Namespaces.RDF + "type"); if (resource == null || !resource.isResource()) return true; if (resource.hasProperty(type_property, class_resource)) return true; else return false; } private static void testGetSourceByUri() { String uri = "http://isi.edu/integration/karma/sources/AEEA5A9A-744C-8096-B372-836ACC820D5A#"; // String uri = "http://isi.edu/integration/karma/sources/940466A8-8733-47B1-2597-11DC112F0437F#"; DataSource source = (DataSource) DataSourceLoader.getInstance().getSourceByUri(uri); if (source != null) source.print(); } private static void testGetAllSources() { List<Source> sourceList = DataSourceLoader.getInstance().getSourcesDetailedInfo(null); for (Source s : sourceList) { if (s != null) s.print(); } } private static void testGetSourcesByIOPattern() { edu.isi.karma.rep.model.Model semanticModel = new edu.isi.karma.rep.model.Model(null); String geonamesOntology = "http://www.geonames.org/ontology#"; String wgs84Ontology = "http://www.w3.org/2003/01/geo/wgs84_pos#"; Label featurePredicatName = new Label(geonamesOntology + "Feature", geonamesOntology, "gn"); Label latPredicatName = new Label(wgs84Ontology + "lat", wgs84Ontology, "wgs84"); Label lngPredicatName = new Label(wgs84Ontology + "long", wgs84Ontology, "wgs84"); ClassAtom c1 = new ClassAtom(featurePredicatName, new Argument("arg1", "arg1", ArgumentType.ATTRIBUTE)); IndividualPropertyAtom p1 = new IndividualPropertyAtom(latPredicatName, new Argument("arg1", "arg1", ArgumentType.ATTRIBUTE), new Argument("arg2", "arg2", ArgumentType.ATTRIBUTE)); IndividualPropertyAtom p2 = new IndividualPropertyAtom(lngPredicatName, new Argument("arg1", "arg1", ArgumentType.ATTRIBUTE), new Argument("arg3", "arg3", ArgumentType.ATTRIBUTE)); // ClassAtom c2 = new ClassAtom(featurePredicatName, new Argument("arg2", "arg2", ArgumentType.ATTRIBUTE)); semanticModel.getAtoms().add(c1); semanticModel.getAtoms().add(p1); semanticModel.getAtoms().add(p2); // semanticModel.getAtoms().add(c2); Map<DataSource, Map<String, String>> sourcesAndMappings = DataSourceLoader.getInstance().getDataSourcesByIOPattern(semanticModel, null); if (sourcesAndMappings == null) return; for (Source s : sourcesAndMappings.keySet()) { if (s != null) s.print(); } System.out.println("Mappings from matched source to model arguments:"); for (Source s : sourcesAndMappings.keySet()) { System.out.println("Source: " + s.getId()); if (sourcesAndMappings.get(s) == null) continue; for (String str : sourcesAndMappings.get(s).keySet()) System.out.println(str + "-------" + sourcesAndMappings.get(s).get(str)); } } private static void testDeleteSourceByUri() { String uri = "http://isi.edu/integration/karma/sources/AEEA5A9A-744C-8096-B372-836ACC820D5A#"; DataSourceLoader.getInstance().deleteSourceByUri(uri); } public static void main(String[] args) { boolean test1 = true, test2 = false, test3 = false, test4 = false; if (test1) testGetSourceByUri(); if (test2) testDeleteSourceByUri(); if (test3) testGetSourcesByIOPattern(); if (test4) testGetAllSources(); } }