/*
* Copyright (c) 2012 Data Harmonisation Panel
*
* All rights reserved. This program and the accompanying materials are made
* available under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 3 of the License,
* or (at your option) any later version.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this distribution. If not, see <http://www.gnu.org/licenses/>.
*
* Contributors:
* HUMBOLDT EU Integrated Project #030962
* Data Harmonisation Panel <http://www.dhpanel.eu>
*/
package eu.esdihumboldt.hale.io.jdbc;
import static eu.esdihumboldt.hale.io.jdbc.JDBCUtil.quote;
import static eu.esdihumboldt.hale.io.jdbc.JDBCUtil.unquote;
import java.io.IOException;
import java.net.URI;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import javax.xml.namespace.QName;
import com.vividsolutions.jts.geom.Geometry;
import eu.esdihumboldt.hale.common.core.io.IOProviderConfigurationException;
import eu.esdihumboldt.hale.common.core.io.ProgressIndicator;
import eu.esdihumboldt.hale.common.core.io.Value;
import eu.esdihumboldt.hale.common.core.io.impl.AbstractIOProvider;
import eu.esdihumboldt.hale.common.core.io.report.IOReporter;
import eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl;
import eu.esdihumboldt.hale.common.schema.SchemaSpaceID;
import eu.esdihumboldt.hale.common.schema.geometry.GeometryProperty;
import eu.esdihumboldt.hale.common.schema.model.ChildDefinition;
import eu.esdihumboldt.hale.common.schema.model.Schema;
import eu.esdihumboldt.hale.common.schema.model.TypeDefinition;
import eu.esdihumboldt.hale.common.schema.model.constraint.property.AutoGenerated;
import eu.esdihumboldt.hale.common.schema.model.constraint.property.Cardinality;
import eu.esdihumboldt.hale.common.schema.model.constraint.property.NillableFlag;
import eu.esdihumboldt.hale.common.schema.model.constraint.property.Reference;
import eu.esdihumboldt.hale.common.schema.model.constraint.type.Binding;
import eu.esdihumboldt.hale.common.schema.model.constraint.type.GeometryType;
import eu.esdihumboldt.hale.common.schema.model.constraint.type.HasValueFlag;
import eu.esdihumboldt.hale.common.schema.model.constraint.type.MappableFlag;
import eu.esdihumboldt.hale.common.schema.model.constraint.type.MappingRelevantFlag;
import eu.esdihumboldt.hale.common.schema.model.impl.DefaultPropertyDefinition;
import eu.esdihumboldt.hale.common.schema.model.impl.DefaultSchema;
import eu.esdihumboldt.hale.common.schema.model.impl.DefaultTypeDefinition;
import eu.esdihumboldt.hale.common.schema.persist.AbstractCachedSchemaReader;
import eu.esdihumboldt.hale.io.jdbc.constraints.AutoIncrementFlag;
import eu.esdihumboldt.hale.io.jdbc.constraints.DatabaseTable;
import eu.esdihumboldt.hale.io.jdbc.constraints.DefaultValue;
import eu.esdihumboldt.hale.io.jdbc.constraints.SQLArray;
import eu.esdihumboldt.hale.io.jdbc.constraints.SQLQuery;
import eu.esdihumboldt.hale.io.jdbc.constraints.SQLType;
import eu.esdihumboldt.hale.io.jdbc.extension.JDBCSchemaReaderAdvisor;
import eu.esdihumboldt.hale.io.jdbc.extension.internal.CustomType;
import eu.esdihumboldt.hale.io.jdbc.extension.internal.CustomTypeExtension;
import eu.esdihumboldt.hale.io.jdbc.extension.internal.GeometryTypeExtension;
import eu.esdihumboldt.hale.io.jdbc.extension.internal.GeometryTypeInfo;
import eu.esdihumboldt.hale.io.jdbc.extension.internal.SchemaReaderAdvisorExtension;
import schemacrawler.schema.BaseColumn;
import schemacrawler.schema.Catalog;
import schemacrawler.schema.Column;
import schemacrawler.schema.ColumnDataType;
//import schemacrawler.schema.Database;
import schemacrawler.schema.IndexColumn;
import schemacrawler.schema.PrimaryKey;
import schemacrawler.schema.ResultsColumn;
import schemacrawler.schema.ResultsColumns;
import schemacrawler.schema.Table;
import schemacrawler.schemacrawler.RegularExpressionInclusionRule;
import schemacrawler.schemacrawler.SchemaCrawlerException;
import schemacrawler.schemacrawler.SchemaCrawlerOptions;
import schemacrawler.schemacrawler.SchemaInfoLevel;
import schemacrawler.utility.SchemaCrawlerUtility;
/**
* Reads a database schema through JDBC.
*
* @author Simon Templer
*/
public class JDBCSchemaReader extends AbstractCachedSchemaReader
implements JDBCConstants, JDBCProvider {
// public static final String PARAM_SCHEMAS = "schemas";
private boolean useQuote;
/**
* Default constructor
*/
public JDBCSchemaReader() {
super();
addSupportedParameter(PARAM_USER);
addSupportedParameter(PARAM_PASSWORD);
useQuote = true;
}
@Override
protected boolean useCache(Value cache) {
// check if a connection can be established
try {
Connection conn = JDBCConnection.getConnection(this);
conn.close();
return false;
} catch (Exception e) {
// on error, use cache
return true;
}
}
/**
* To set useQuote parameter
*
* @param useQuote true or false value
*/
public void setUseQuotes(boolean useQuote) {
this.useQuote = useQuote;
}
/**
* To get Connection. Override this to load the customized connection
*
* @return Connection object after loading driver.
* @throws SQLException if connection could not be made.
*/
@Override
public Connection getConnection() throws SQLException {
return JDBCConnection.getConnection(this);
}
@Override
protected Schema loadFromSource(ProgressIndicator progress, IOReporter reporter)
throws IOProviderConfigurationException, IOException {
DefaultSchema typeIndex = null;
progress.begin("Read database schema", ProgressIndicator.UNKNOWN);
Connection connection = null;
try {
// connect to the database
try {
connection = getConnection();
} catch (Exception e) {
reporter.error(new IOMessageImpl(e.getLocalizedMessage(), e));
reporter.setSuccess(false);
reporter.setSummary("Failed to connect to database.");
return null;
}
// don't fail if Connection.setReadOnly() throws an exception (e.g.
// SQLite JDBC driver does not allow changing the flag after the
// connection has been created), report a warning message instead
try {
connection.setReadOnly(true);
} catch (SQLException e) {
// ignore
// reporter.warn(new IOMessageImpl(e.getLocalizedMessage(), e));
}
URI jdbcURI = getSource().getLocation();
final SchemaCrawlerOptions options = new SchemaCrawlerOptions();
SchemaInfoLevel level = new SchemaInfoLevel();
level.setTag("hale");
// these are enabled by default, we don't need them (yet)
level.setRetrieveSchemaCrawlerInfo(false);
level.setRetrieveJdbcDriverInfo(false);
level.setRetrieveDatabaseInfo(false);
// set what we need
level.setRetrieveTables(true);
level.setRetrieveColumnDataTypes(true);
level.setRetrieveUserDefinedColumnDataTypes(true);
level.setRetrieveTableColumns(true); // to get table columns
// information, also
// includes primary key
level.setRetrieveForeignKeys(true); // to get linking information
// level.setRetrieveIndices(true); // to get info about UNIQUE indices for validation
// XXX For some advanced info / DBMS specific info we'll need a
// properties file. See Config & InformationSchemaViews.
level.setTag("hale");
if (getParameter(SCHEMAS).as(String.class) != null) {
String schemas = getParameter(SCHEMAS).as(String.class).replace(',', '|');
options.setSchemaInclusionRule(new RegularExpressionInclusionRule(schemas));
}
if (SchemaSpaceID.SOURCE.equals(getSchemaSpace())) {
// show views and tables
List<String> tableList = Arrays.asList("TABLE", "VIEW");
options.setTableTypes(tableList);
}
else {
// only show tables
options.setTableTypes(Arrays.asList("TABLE"));
}
options.setSchemaInfoLevel(level);
// get advisor
// XXX should be created once, and used in other places if
// applicable
JDBCSchemaReaderAdvisor advisor = SchemaReaderAdvisorExtension.getInstance()
.getAdvisor(connection);
if (advisor != null) {
advisor.configureSchemaCrawler(options);
}
final Catalog database = SchemaCrawlerUtility.getCatalog(connection, options);
@SuppressWarnings("unused")
String quotes = JDBCUtil.determineQuoteString(connection);
// FIXME not actually used here or in SQL schema reader
String overallNamespace = JDBCUtil.determineNamespace(jdbcURI, advisor);
// create the type index
typeIndex = new DefaultSchema(overallNamespace, jdbcURI);
for (final schemacrawler.schema.Schema schema : database.getSchemas()) {
// each schema represents a namespace
String namespace;
if (overallNamespace.isEmpty()) {
namespace = unquote(schema.getName());
}
else {
namespace = overallNamespace;
if (schema.getName() != null) {
namespace += ":" + unquote(schema.getName());
}
}
for (final Table table : database.getTables(schema)) {
// each table is a type
// get the type definition
TypeDefinition type = getOrCreateTableType(schema, table, overallNamespace,
namespace, typeIndex, connection, reporter, database);
// get ResultSetMetaData for extra info about columns (e. g.
// auto increment)
ResultsColumns additionalInfo = null;
Statement stmt = null;
try {
stmt = connection.createStatement();
// get if in table name, quotation required or not.
String fullTableName = getQuotedValue(table.getName());
if (schema.getName() != null) {
fullTableName = getQuotedValue(schema.getName()) + "." + fullTableName;
}
ResultSet rs = stmt
.executeQuery("SELECT * FROM " + fullTableName + " WHERE 1 = 0");
additionalInfo = SchemaCrawlerUtility.getResultColumns(rs);
} catch (SQLException sqle) {
reporter.warn(new IOMessageImpl(
"Couldn't retrieve additional column meta data.", sqle));
} finally {
if (stmt != null)
try {
stmt.close();
} catch (SQLException e) {
// ignore
}
}
// create property definitions for each column
for (final Column column : table.getColumns()) {
DefaultPropertyDefinition property = getOrCreateProperty(schema, type,
column, overallNamespace, namespace, typeIndex, connection,
reporter, database);
// Set auto increment flag if meta data says so.
// Not sure, whether that covers every case of
// "auto increment" for every DBMS (probably not).
// But there is nothing else we can do in general
// without lots of DBMS specific code or asking the user
// for input.
// XXX does not work for example for PostgreSQL
if (additionalInfo != null) {
// ResultColumns does not quote the column namen in
// contrast to every other place
ResultsColumn rc = additionalInfo.getColumn(unquote(column.getName()));
if (rc != null && rc.isAutoIncrement())
property.setConstraint(AutoIncrementFlag.get(true));
}
}
}
}
reporter.setSuccess(true);
} catch (SchemaCrawlerException e) {
throw new IOProviderConfigurationException("Failed to read database schema", e);
} finally {
if (connection != null) {
try {
connection.close();
} catch (SQLException e) {
// ignore
}
}
progress.end();
}
return typeIndex;
}
/**
* Get quoted value by deciding on isSchemaNameQuoted parameter.
*
* @param value String
* @return quoted unqoted string
*/
private String getQuotedValue(String value) {
if (useQuote) {
value = quote(value);
}
return value;
}
/**
* Gets or creates a property definition for the given column. Its type
* definition is created, too, if necessary.
*
* @param schema the schema the table belongs to
* @param tableType the type definition of the parent table this column
* belongs too
* @param column the column to get or create a property definition for
* @param overallNamespace the database namespace
* @param namespace the schema namespace
* @param typeIndex the type index
* @param connection the database connection
* @param reporter the reporter
* @param catalog the catalog for access to other column types
* @return the property definition for the given column
*/
private DefaultPropertyDefinition getOrCreateProperty(schemacrawler.schema.Schema schema,
TypeDefinition tableType, Column column, String overallNamespace, String namespace,
DefaultSchema typeIndex, Connection connection, IOReporter reporter, Catalog catalog) {
QName name = new QName(unquote(column.getName()));
// check for existing property definition
ChildDefinition<?> existing = tableType.getChild(name);
if (existing != null) {
return (DefaultPropertyDefinition) existing;
}
// create new one
// determine the column type
TypeDefinition columnType = getOrCreateColumnType(column, overallNamespace, typeIndex,
connection, tableType, reporter, catalog);
SQLArray arrayInfo = columnType.getConstraint(SQLArray.class);
// create the property
DefaultPropertyDefinition property = new DefaultPropertyDefinition(name, tableType,
columnType);
// configure property
if (column.getRemarks() != null && !column.getRemarks().isEmpty()) {
property.setDescription(column.getRemarks());
}
property.setConstraint(NillableFlag.get(column.isNullable()));
// XXX Default value is read as string from the meta data.
// This is probably not really a problem, but should be noted!
// XXX In particular the default value can be a function call like for
// example GETDATE().
String defaultValue = column.getDefaultValue();
if (arrayInfo.isArray() && arrayInfo.getDimension() <= 1) {
// XXX for now, use multiple occurrence representation also if
// dimension is not known (0)
if (!arrayInfo.hasSize(0)) {
property.setConstraint(Cardinality.CC_ANY_NUMBER);
}
else {
long min = 0; // XXX what is appropriate as minimum?
long max = arrayInfo.getSize(0);
property.setConstraint(Cardinality.get(min, max));
}
}
else if (defaultValue != null) {
property.setConstraint(new DefaultValue(defaultValue));
property.setConstraint(Cardinality.CC_OPTIONAL);
}
else
property.setConstraint(Cardinality.CC_EXACTLY_ONCE);
// set auto incremented constraint which denotes if IDs are auto
// incremented or not
if (column.isAutoIncremented()) {
property.setConstraint(AutoGenerated.get(true));
}
else {
property.setConstraint(AutoGenerated.get(false));
}
// XXX constraint for column.isPartOfPrimaryKey(),
// column.isPartOfUniqueIndex()
// XXX what if the foreign key consists of multiple columns?
// those indices/foreign keys should maybe belong to the table type,
// since they can have multiple columns
if (column.isPartOfForeignKey()) {
Column referenced = column.getReferencedColumn();
// got this case in MSSQL, foreign key on the same column of the
// same table because of that it goes in to loop and throws
// StackOverFlow exception.
if (!(referenced.getParent().equals(column.getParent()) && referenced.equals(column))) {
// Referenced table can be in different schema.
// creation of referenced column's table should not be with same
// Schema or Namespace. It should be with referenced table's
// Schema and namespace.
String referencedNameSpace = getReferencedTableNamespace(referenced.getParent(),
overallNamespace);
property.setConstraint(
new Reference(getOrCreateTableType(referenced.getParent().getSchema(),
referenced.getParent(), overallNamespace, referencedNameSpace,
typeIndex, connection, reporter, catalog)));
}
}
return property;
}
/**
* Get or create the type definition for the given column.
*
* @param column the column
* @param overallNamespace the database namespace
* @param types the type index
* @param connection the database connection
* @param tableType the type definition of the table the column is part of
* @param reporter the reporter
* @param catalog the catalog for access to other column types
* @return the type definition for the column type
*/
public static TypeDefinition getOrCreateColumnType(BaseColumn<?> column,
final String overallNamespace, DefaultSchema types, Connection connection,
TypeDefinition tableType, IOReporter reporter, Catalog catalog) {
// XXX what about shared types?
// TODO the size/width info (VARCHAR(_30_)) is in column, the
// columntype/-name is not sufficient
ColumnDataType columnType = column.getColumnDataType();// getType();
String localName = columnType.getName();
QName typeName = new QName(overallNamespace, localName);
// check for geometry type
GeometryTypeInfo geomType = GeometryTypeExtension.getInstance()
.getTypeInfo(columnType.getName(), connection);
@SuppressWarnings("rawtypes")
GeometryAdvisor geomAdvisor = null;
if (geomType != null) {
geomAdvisor = geomType.getGeometryAdvisor();
// determine if a type specifically for this column is needed
if (!geomAdvisor.isFixedType(columnType)) {
// must use a specific type definition for this column
// -> use a type name based on the column
// new namespace is the table and column name
String ns = tableType.getName().getNamespaceURI() + '/'
+ tableType.getName().getLocalPart() + '/' + unquote(column.getName());
typeName = new QName(ns, localName);
}
}
// check for existing type
TypeDefinition existing = types.getType(typeName);
if (existing != null)
return existing;
// create new type
DefaultTypeDefinition type = new DefaultTypeDefinition(typeName);
type.setConstraint(HasValueFlag.ENABLED);
CustomType cust = CustomTypeExtension.getInstance().getCustomType(localName, connection);
/*
* Oracle jdbc was returning sqltype -6 for NUMBER data type, but it is
* bound to boolean by Types.java class. Configured the sqltype 6 for
* NUMBER data type.
*/
if (cust != null) {
type.setConstraint(SQLType.get(cust.getSQLType()));
}
else {
type.setConstraint(SQLType.get(columnType.getJavaSqlType().getJavaSqlType()));
}
if (geomType != null && geomAdvisor != null) {
// configure geometry type
@SuppressWarnings("unchecked")
Class<? extends Geometry> geomClass = geomAdvisor
.configureGeometryColumnType(connection, column, type);
type.setConstraint(GeometryType.get(geomClass));
// always a single geometry
type.setConstraint(Binding.get(GeometryProperty.class));
// remember advisor for type (used in instance writer)
type.setConstraint(geomType.getConstraint());
}
else {
// configure type
Class<?> binding = null;
if (cust != null) {
binding = cust.getBinding();
}
else {
if (column.getColumnDataType().getJavaSqlType().getJavaSqlType() == Types.ARRAY) {
// TODO let this be handled by a possible advisor?
/*
* Special handling for arrays.
*
* Challenges:
*
* - find the type of contained items
*
* - determine dimensions and size
*/
Class<?> elementBinding;
// determine type/binding of contained items
ColumnDataType cdt = column.getColumnDataType();
ColumnDataType itemType = null;
String dbTypeName = cdt.getDatabaseSpecificTypeName();
// XXX special approach for Postgres - strip underscore
// prefix and look up type
if (dbTypeName.startsWith("_")) {
String testName = dbTypeName.substring(1);
itemType = catalog.getSystemColumnDataType(testName);
}
if (itemType == null) {
// generic binding
elementBinding = Object.class;
reporter.error(new IOMessageImpl(
"Could not determine element type for array column", null));
}
else {
elementBinding = itemType.getTypeMappedClass();
// TODO support custom bindings?
// XXX probably needed for Oracle?
}
// dimensions and size cannot be determined from schema
// crawler it seems - would need database specific queries
// (if the info is available at all)
/*
* Postgres:
*
* Dimensions and size are not part of the schema, they can
* only be determined for a value.
*/
// XXX for now, stick to what we can determine
int dimension = SQLArray.UNKNOWN_DIMENSION;
String specificTypeName = (itemType != null)
? (itemType.getDatabaseSpecificTypeName()) : (null);
type.setConstraint(
new SQLArray(elementBinding, specificTypeName, dimension, null));
// set binding
if (dimension <= 1) {
// XXX for now, use this representation also if
// dimension is not known
// 1-dimensional -> as multiple occurrences
binding = elementBinding;
}
else {
// XXX use collection or something similar instead?
binding = Object.class;
}
}
else {
binding = column.getColumnDataType().getTypeMappedClass();
}
}
type.setConstraint(Binding.get(binding));
type.setConstraint(HasValueFlag.ENABLED);
}
// TODO validation constraint
if (columnType.getRemarks() != null && !columnType.getRemarks().isEmpty())
type.setDescription(columnType.getRemarks());
types.addType(type);
return type;
}
@SuppressWarnings("unused")
private Class<?> loadColumnBinding(String name) throws ClassNotFoundException {
switch (name) {
case "byte":
return byte.class;
case "int":
return int.class;
case "long":
return long.class;
case "boolean":
return boolean.class;
default:
return Class.forName(name);
}
}
/**
* Get or create the type definition for the given table.
*
* @param schema the schema in which the table exists
* @param table the table
* @param overallNamespace the database namespace
* @param namespace the schema namespace
* @param types the type index
* @param connection the database connection
* @param reporter the reporter
* @param catalog the catalog for access to other column types
* @return the type definition for the given table
*/
private TypeDefinition getOrCreateTableType(schemacrawler.schema.Schema schema, Table table,
String overallNamespace, String namespace, DefaultSchema types, Connection connection,
IOReporter reporter, Catalog catalog) {
QName typeName = new QName(namespace, unquote(table.getName()));
// check for existing type
TypeDefinition existingType = types.getType(typeName);
if (existingType != null)
return existingType;
// create new type
DefaultTypeDefinition type = new DefaultTypeDefinition(typeName);
// set description if available
if (table.getRemarks() != null && !table.getRemarks().isEmpty())
type.setDescription(table.getRemarks());
// configure type
type.setConstraint(MappableFlag.ENABLED);
type.setConstraint(MappingRelevantFlag.ENABLED);
type.setConstraint(HasValueFlag.DISABLED);
// set schema and table name
DatabaseTable tableConstraint = new DatabaseTable(unquote(schema.getName()),
unquote(table.getName()), useQuote);
type.setConstraint(tableConstraint);
// set SQL query constraint
String query = "SELECT * FROM " + tableConstraint.getFullTableName();
type.setConstraint(new SQLQuery(query));
// set primary key if possible
PrimaryKey key = null;
try {
key = table.getPrimaryKey();
} catch (Exception e) {
reporter.warn(new IOMessageImpl(
"Could not read primary key metadata for table: " + table.getFullName(), e));
}
if (key != null) {
List<IndexColumn> columns = key.getColumns();
if (columns.size() > 1) {
reporter.warn(new IOMessageImpl(
"Primary keys over multiple columns are not yet supported.", null));
}
else if (columns.size() == 1) {
// create constraint, get property definition for original table
// column (maybe could use index column, too)
type.setConstraint(
new eu.esdihumboldt.hale.common.schema.model.constraint.type.PrimaryKey(
Collections.<QName> singletonList(getOrCreateProperty(schema, type,
table.getColumn(columns.get(0).getName()), overallNamespace,
namespace, types, connection, reporter, catalog)
.getName())));
}
}
// TODO validation of constraints? Most are about several instances (i.
// e. UNIQUE)
types.addType(type);
return type;
}
/**
* @see AbstractIOProvider#getDefaultTypeName()
*/
@Override
protected String getDefaultTypeName() {
return "Database";
}
/**
* Get namespace of the referenced table
*
* @param referencedTable a {@link Table}
* @param overallNamespace the database namespace
* @return Namespace of referenced table
*/
private String getReferencedTableNamespace(Table referencedTable, String overallNamespace) {
String namespace;
if (overallNamespace.isEmpty()) {
namespace = unquote(referencedTable.getSchema().getName());
}
else {
namespace = overallNamespace;
if (referencedTable.getSchema().getName() != null) {
namespace += ":" + unquote(referencedTable.getSchema().getName());
}
}
return namespace;
}
}