/* * Copyright (c) 2012 Data Harmonisation Panel * * All rights reserved. This program and the accompanying materials are made * available under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation, either version 3 of the License, * or (at your option) any later version. * * You should have received a copy of the GNU Lesser General Public License * along with this distribution. If not, see <http://www.gnu.org/licenses/>. * * Contributors: * HUMBOLDT EU Integrated Project #030962 * Data Harmonisation Panel <http://www.dhpanel.eu> */ package eu.esdihumboldt.hale.io.jdbc; import static eu.esdihumboldt.hale.io.jdbc.JDBCUtil.quote; import static eu.esdihumboldt.hale.io.jdbc.JDBCUtil.unquote; import java.io.IOException; import java.net.URI; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; import java.util.Arrays; import java.util.Collections; import java.util.List; import javax.xml.namespace.QName; import com.vividsolutions.jts.geom.Geometry; import eu.esdihumboldt.hale.common.core.io.IOProviderConfigurationException; import eu.esdihumboldt.hale.common.core.io.ProgressIndicator; import eu.esdihumboldt.hale.common.core.io.Value; import eu.esdihumboldt.hale.common.core.io.impl.AbstractIOProvider; import eu.esdihumboldt.hale.common.core.io.report.IOReporter; import eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl; import eu.esdihumboldt.hale.common.schema.SchemaSpaceID; import eu.esdihumboldt.hale.common.schema.geometry.GeometryProperty; import eu.esdihumboldt.hale.common.schema.model.ChildDefinition; import eu.esdihumboldt.hale.common.schema.model.Schema; import eu.esdihumboldt.hale.common.schema.model.TypeDefinition; import eu.esdihumboldt.hale.common.schema.model.constraint.property.AutoGenerated; import eu.esdihumboldt.hale.common.schema.model.constraint.property.Cardinality; import eu.esdihumboldt.hale.common.schema.model.constraint.property.NillableFlag; import eu.esdihumboldt.hale.common.schema.model.constraint.property.Reference; import eu.esdihumboldt.hale.common.schema.model.constraint.type.Binding; import eu.esdihumboldt.hale.common.schema.model.constraint.type.GeometryType; import eu.esdihumboldt.hale.common.schema.model.constraint.type.HasValueFlag; import eu.esdihumboldt.hale.common.schema.model.constraint.type.MappableFlag; import eu.esdihumboldt.hale.common.schema.model.constraint.type.MappingRelevantFlag; import eu.esdihumboldt.hale.common.schema.model.impl.DefaultPropertyDefinition; import eu.esdihumboldt.hale.common.schema.model.impl.DefaultSchema; import eu.esdihumboldt.hale.common.schema.model.impl.DefaultTypeDefinition; import eu.esdihumboldt.hale.common.schema.persist.AbstractCachedSchemaReader; import eu.esdihumboldt.hale.io.jdbc.constraints.AutoIncrementFlag; import eu.esdihumboldt.hale.io.jdbc.constraints.DatabaseTable; import eu.esdihumboldt.hale.io.jdbc.constraints.DefaultValue; import eu.esdihumboldt.hale.io.jdbc.constraints.SQLArray; import eu.esdihumboldt.hale.io.jdbc.constraints.SQLQuery; import eu.esdihumboldt.hale.io.jdbc.constraints.SQLType; import eu.esdihumboldt.hale.io.jdbc.extension.JDBCSchemaReaderAdvisor; import eu.esdihumboldt.hale.io.jdbc.extension.internal.CustomType; import eu.esdihumboldt.hale.io.jdbc.extension.internal.CustomTypeExtension; import eu.esdihumboldt.hale.io.jdbc.extension.internal.GeometryTypeExtension; import eu.esdihumboldt.hale.io.jdbc.extension.internal.GeometryTypeInfo; import eu.esdihumboldt.hale.io.jdbc.extension.internal.SchemaReaderAdvisorExtension; import schemacrawler.schema.BaseColumn; import schemacrawler.schema.Catalog; import schemacrawler.schema.Column; import schemacrawler.schema.ColumnDataType; //import schemacrawler.schema.Database; import schemacrawler.schema.IndexColumn; import schemacrawler.schema.PrimaryKey; import schemacrawler.schema.ResultsColumn; import schemacrawler.schema.ResultsColumns; import schemacrawler.schema.Table; import schemacrawler.schemacrawler.RegularExpressionInclusionRule; import schemacrawler.schemacrawler.SchemaCrawlerException; import schemacrawler.schemacrawler.SchemaCrawlerOptions; import schemacrawler.schemacrawler.SchemaInfoLevel; import schemacrawler.utility.SchemaCrawlerUtility; /** * Reads a database schema through JDBC. * * @author Simon Templer */ public class JDBCSchemaReader extends AbstractCachedSchemaReader implements JDBCConstants, JDBCProvider { // public static final String PARAM_SCHEMAS = "schemas"; private boolean useQuote; /** * Default constructor */ public JDBCSchemaReader() { super(); addSupportedParameter(PARAM_USER); addSupportedParameter(PARAM_PASSWORD); useQuote = true; } @Override protected boolean useCache(Value cache) { // check if a connection can be established try { Connection conn = JDBCConnection.getConnection(this); conn.close(); return false; } catch (Exception e) { // on error, use cache return true; } } /** * To set useQuote parameter * * @param useQuote true or false value */ public void setUseQuotes(boolean useQuote) { this.useQuote = useQuote; } /** * To get Connection. Override this to load the customized connection * * @return Connection object after loading driver. * @throws SQLException if connection could not be made. */ @Override public Connection getConnection() throws SQLException { return JDBCConnection.getConnection(this); } @Override protected Schema loadFromSource(ProgressIndicator progress, IOReporter reporter) throws IOProviderConfigurationException, IOException { DefaultSchema typeIndex = null; progress.begin("Read database schema", ProgressIndicator.UNKNOWN); Connection connection = null; try { // connect to the database try { connection = getConnection(); } catch (Exception e) { reporter.error(new IOMessageImpl(e.getLocalizedMessage(), e)); reporter.setSuccess(false); reporter.setSummary("Failed to connect to database."); return null; } // don't fail if Connection.setReadOnly() throws an exception (e.g. // SQLite JDBC driver does not allow changing the flag after the // connection has been created), report a warning message instead try { connection.setReadOnly(true); } catch (SQLException e) { // ignore // reporter.warn(new IOMessageImpl(e.getLocalizedMessage(), e)); } URI jdbcURI = getSource().getLocation(); final SchemaCrawlerOptions options = new SchemaCrawlerOptions(); SchemaInfoLevel level = new SchemaInfoLevel(); level.setTag("hale"); // these are enabled by default, we don't need them (yet) level.setRetrieveSchemaCrawlerInfo(false); level.setRetrieveJdbcDriverInfo(false); level.setRetrieveDatabaseInfo(false); // set what we need level.setRetrieveTables(true); level.setRetrieveColumnDataTypes(true); level.setRetrieveUserDefinedColumnDataTypes(true); level.setRetrieveTableColumns(true); // to get table columns // information, also // includes primary key level.setRetrieveForeignKeys(true); // to get linking information // level.setRetrieveIndices(true); // to get info about UNIQUE indices for validation // XXX For some advanced info / DBMS specific info we'll need a // properties file. See Config & InformationSchemaViews. level.setTag("hale"); if (getParameter(SCHEMAS).as(String.class) != null) { String schemas = getParameter(SCHEMAS).as(String.class).replace(',', '|'); options.setSchemaInclusionRule(new RegularExpressionInclusionRule(schemas)); } if (SchemaSpaceID.SOURCE.equals(getSchemaSpace())) { // show views and tables List<String> tableList = Arrays.asList("TABLE", "VIEW"); options.setTableTypes(tableList); } else { // only show tables options.setTableTypes(Arrays.asList("TABLE")); } options.setSchemaInfoLevel(level); // get advisor // XXX should be created once, and used in other places if // applicable JDBCSchemaReaderAdvisor advisor = SchemaReaderAdvisorExtension.getInstance() .getAdvisor(connection); if (advisor != null) { advisor.configureSchemaCrawler(options); } final Catalog database = SchemaCrawlerUtility.getCatalog(connection, options); @SuppressWarnings("unused") String quotes = JDBCUtil.determineQuoteString(connection); // FIXME not actually used here or in SQL schema reader String overallNamespace = JDBCUtil.determineNamespace(jdbcURI, advisor); // create the type index typeIndex = new DefaultSchema(overallNamespace, jdbcURI); for (final schemacrawler.schema.Schema schema : database.getSchemas()) { // each schema represents a namespace String namespace; if (overallNamespace.isEmpty()) { namespace = unquote(schema.getName()); } else { namespace = overallNamespace; if (schema.getName() != null) { namespace += ":" + unquote(schema.getName()); } } for (final Table table : database.getTables(schema)) { // each table is a type // get the type definition TypeDefinition type = getOrCreateTableType(schema, table, overallNamespace, namespace, typeIndex, connection, reporter, database); // get ResultSetMetaData for extra info about columns (e. g. // auto increment) ResultsColumns additionalInfo = null; Statement stmt = null; try { stmt = connection.createStatement(); // get if in table name, quotation required or not. String fullTableName = getQuotedValue(table.getName()); if (schema.getName() != null) { fullTableName = getQuotedValue(schema.getName()) + "." + fullTableName; } ResultSet rs = stmt .executeQuery("SELECT * FROM " + fullTableName + " WHERE 1 = 0"); additionalInfo = SchemaCrawlerUtility.getResultColumns(rs); } catch (SQLException sqle) { reporter.warn(new IOMessageImpl( "Couldn't retrieve additional column meta data.", sqle)); } finally { if (stmt != null) try { stmt.close(); } catch (SQLException e) { // ignore } } // create property definitions for each column for (final Column column : table.getColumns()) { DefaultPropertyDefinition property = getOrCreateProperty(schema, type, column, overallNamespace, namespace, typeIndex, connection, reporter, database); // Set auto increment flag if meta data says so. // Not sure, whether that covers every case of // "auto increment" for every DBMS (probably not). // But there is nothing else we can do in general // without lots of DBMS specific code or asking the user // for input. // XXX does not work for example for PostgreSQL if (additionalInfo != null) { // ResultColumns does not quote the column namen in // contrast to every other place ResultsColumn rc = additionalInfo.getColumn(unquote(column.getName())); if (rc != null && rc.isAutoIncrement()) property.setConstraint(AutoIncrementFlag.get(true)); } } } } reporter.setSuccess(true); } catch (SchemaCrawlerException e) { throw new IOProviderConfigurationException("Failed to read database schema", e); } finally { if (connection != null) { try { connection.close(); } catch (SQLException e) { // ignore } } progress.end(); } return typeIndex; } /** * Get quoted value by deciding on isSchemaNameQuoted parameter. * * @param value String * @return quoted unqoted string */ private String getQuotedValue(String value) { if (useQuote) { value = quote(value); } return value; } /** * Gets or creates a property definition for the given column. Its type * definition is created, too, if necessary. * * @param schema the schema the table belongs to * @param tableType the type definition of the parent table this column * belongs too * @param column the column to get or create a property definition for * @param overallNamespace the database namespace * @param namespace the schema namespace * @param typeIndex the type index * @param connection the database connection * @param reporter the reporter * @param catalog the catalog for access to other column types * @return the property definition for the given column */ private DefaultPropertyDefinition getOrCreateProperty(schemacrawler.schema.Schema schema, TypeDefinition tableType, Column column, String overallNamespace, String namespace, DefaultSchema typeIndex, Connection connection, IOReporter reporter, Catalog catalog) { QName name = new QName(unquote(column.getName())); // check for existing property definition ChildDefinition<?> existing = tableType.getChild(name); if (existing != null) { return (DefaultPropertyDefinition) existing; } // create new one // determine the column type TypeDefinition columnType = getOrCreateColumnType(column, overallNamespace, typeIndex, connection, tableType, reporter, catalog); SQLArray arrayInfo = columnType.getConstraint(SQLArray.class); // create the property DefaultPropertyDefinition property = new DefaultPropertyDefinition(name, tableType, columnType); // configure property if (column.getRemarks() != null && !column.getRemarks().isEmpty()) { property.setDescription(column.getRemarks()); } property.setConstraint(NillableFlag.get(column.isNullable())); // XXX Default value is read as string from the meta data. // This is probably not really a problem, but should be noted! // XXX In particular the default value can be a function call like for // example GETDATE(). String defaultValue = column.getDefaultValue(); if (arrayInfo.isArray() && arrayInfo.getDimension() <= 1) { // XXX for now, use multiple occurrence representation also if // dimension is not known (0) if (!arrayInfo.hasSize(0)) { property.setConstraint(Cardinality.CC_ANY_NUMBER); } else { long min = 0; // XXX what is appropriate as minimum? long max = arrayInfo.getSize(0); property.setConstraint(Cardinality.get(min, max)); } } else if (defaultValue != null) { property.setConstraint(new DefaultValue(defaultValue)); property.setConstraint(Cardinality.CC_OPTIONAL); } else property.setConstraint(Cardinality.CC_EXACTLY_ONCE); // set auto incremented constraint which denotes if IDs are auto // incremented or not if (column.isAutoIncremented()) { property.setConstraint(AutoGenerated.get(true)); } else { property.setConstraint(AutoGenerated.get(false)); } // XXX constraint for column.isPartOfPrimaryKey(), // column.isPartOfUniqueIndex() // XXX what if the foreign key consists of multiple columns? // those indices/foreign keys should maybe belong to the table type, // since they can have multiple columns if (column.isPartOfForeignKey()) { Column referenced = column.getReferencedColumn(); // got this case in MSSQL, foreign key on the same column of the // same table because of that it goes in to loop and throws // StackOverFlow exception. if (!(referenced.getParent().equals(column.getParent()) && referenced.equals(column))) { // Referenced table can be in different schema. // creation of referenced column's table should not be with same // Schema or Namespace. It should be with referenced table's // Schema and namespace. String referencedNameSpace = getReferencedTableNamespace(referenced.getParent(), overallNamespace); property.setConstraint( new Reference(getOrCreateTableType(referenced.getParent().getSchema(), referenced.getParent(), overallNamespace, referencedNameSpace, typeIndex, connection, reporter, catalog))); } } return property; } /** * Get or create the type definition for the given column. * * @param column the column * @param overallNamespace the database namespace * @param types the type index * @param connection the database connection * @param tableType the type definition of the table the column is part of * @param reporter the reporter * @param catalog the catalog for access to other column types * @return the type definition for the column type */ public static TypeDefinition getOrCreateColumnType(BaseColumn<?> column, final String overallNamespace, DefaultSchema types, Connection connection, TypeDefinition tableType, IOReporter reporter, Catalog catalog) { // XXX what about shared types? // TODO the size/width info (VARCHAR(_30_)) is in column, the // columntype/-name is not sufficient ColumnDataType columnType = column.getColumnDataType();// getType(); String localName = columnType.getName(); QName typeName = new QName(overallNamespace, localName); // check for geometry type GeometryTypeInfo geomType = GeometryTypeExtension.getInstance() .getTypeInfo(columnType.getName(), connection); @SuppressWarnings("rawtypes") GeometryAdvisor geomAdvisor = null; if (geomType != null) { geomAdvisor = geomType.getGeometryAdvisor(); // determine if a type specifically for this column is needed if (!geomAdvisor.isFixedType(columnType)) { // must use a specific type definition for this column // -> use a type name based on the column // new namespace is the table and column name String ns = tableType.getName().getNamespaceURI() + '/' + tableType.getName().getLocalPart() + '/' + unquote(column.getName()); typeName = new QName(ns, localName); } } // check for existing type TypeDefinition existing = types.getType(typeName); if (existing != null) return existing; // create new type DefaultTypeDefinition type = new DefaultTypeDefinition(typeName); type.setConstraint(HasValueFlag.ENABLED); CustomType cust = CustomTypeExtension.getInstance().getCustomType(localName, connection); /* * Oracle jdbc was returning sqltype -6 for NUMBER data type, but it is * bound to boolean by Types.java class. Configured the sqltype 6 for * NUMBER data type. */ if (cust != null) { type.setConstraint(SQLType.get(cust.getSQLType())); } else { type.setConstraint(SQLType.get(columnType.getJavaSqlType().getJavaSqlType())); } if (geomType != null && geomAdvisor != null) { // configure geometry type @SuppressWarnings("unchecked") Class<? extends Geometry> geomClass = geomAdvisor .configureGeometryColumnType(connection, column, type); type.setConstraint(GeometryType.get(geomClass)); // always a single geometry type.setConstraint(Binding.get(GeometryProperty.class)); // remember advisor for type (used in instance writer) type.setConstraint(geomType.getConstraint()); } else { // configure type Class<?> binding = null; if (cust != null) { binding = cust.getBinding(); } else { if (column.getColumnDataType().getJavaSqlType().getJavaSqlType() == Types.ARRAY) { // TODO let this be handled by a possible advisor? /* * Special handling for arrays. * * Challenges: * * - find the type of contained items * * - determine dimensions and size */ Class<?> elementBinding; // determine type/binding of contained items ColumnDataType cdt = column.getColumnDataType(); ColumnDataType itemType = null; String dbTypeName = cdt.getDatabaseSpecificTypeName(); // XXX special approach for Postgres - strip underscore // prefix and look up type if (dbTypeName.startsWith("_")) { String testName = dbTypeName.substring(1); itemType = catalog.getSystemColumnDataType(testName); } if (itemType == null) { // generic binding elementBinding = Object.class; reporter.error(new IOMessageImpl( "Could not determine element type for array column", null)); } else { elementBinding = itemType.getTypeMappedClass(); // TODO support custom bindings? // XXX probably needed for Oracle? } // dimensions and size cannot be determined from schema // crawler it seems - would need database specific queries // (if the info is available at all) /* * Postgres: * * Dimensions and size are not part of the schema, they can * only be determined for a value. */ // XXX for now, stick to what we can determine int dimension = SQLArray.UNKNOWN_DIMENSION; String specificTypeName = (itemType != null) ? (itemType.getDatabaseSpecificTypeName()) : (null); type.setConstraint( new SQLArray(elementBinding, specificTypeName, dimension, null)); // set binding if (dimension <= 1) { // XXX for now, use this representation also if // dimension is not known // 1-dimensional -> as multiple occurrences binding = elementBinding; } else { // XXX use collection or something similar instead? binding = Object.class; } } else { binding = column.getColumnDataType().getTypeMappedClass(); } } type.setConstraint(Binding.get(binding)); type.setConstraint(HasValueFlag.ENABLED); } // TODO validation constraint if (columnType.getRemarks() != null && !columnType.getRemarks().isEmpty()) type.setDescription(columnType.getRemarks()); types.addType(type); return type; } @SuppressWarnings("unused") private Class<?> loadColumnBinding(String name) throws ClassNotFoundException { switch (name) { case "byte": return byte.class; case "int": return int.class; case "long": return long.class; case "boolean": return boolean.class; default: return Class.forName(name); } } /** * Get or create the type definition for the given table. * * @param schema the schema in which the table exists * @param table the table * @param overallNamespace the database namespace * @param namespace the schema namespace * @param types the type index * @param connection the database connection * @param reporter the reporter * @param catalog the catalog for access to other column types * @return the type definition for the given table */ private TypeDefinition getOrCreateTableType(schemacrawler.schema.Schema schema, Table table, String overallNamespace, String namespace, DefaultSchema types, Connection connection, IOReporter reporter, Catalog catalog) { QName typeName = new QName(namespace, unquote(table.getName())); // check for existing type TypeDefinition existingType = types.getType(typeName); if (existingType != null) return existingType; // create new type DefaultTypeDefinition type = new DefaultTypeDefinition(typeName); // set description if available if (table.getRemarks() != null && !table.getRemarks().isEmpty()) type.setDescription(table.getRemarks()); // configure type type.setConstraint(MappableFlag.ENABLED); type.setConstraint(MappingRelevantFlag.ENABLED); type.setConstraint(HasValueFlag.DISABLED); // set schema and table name DatabaseTable tableConstraint = new DatabaseTable(unquote(schema.getName()), unquote(table.getName()), useQuote); type.setConstraint(tableConstraint); // set SQL query constraint String query = "SELECT * FROM " + tableConstraint.getFullTableName(); type.setConstraint(new SQLQuery(query)); // set primary key if possible PrimaryKey key = null; try { key = table.getPrimaryKey(); } catch (Exception e) { reporter.warn(new IOMessageImpl( "Could not read primary key metadata for table: " + table.getFullName(), e)); } if (key != null) { List<IndexColumn> columns = key.getColumns(); if (columns.size() > 1) { reporter.warn(new IOMessageImpl( "Primary keys over multiple columns are not yet supported.", null)); } else if (columns.size() == 1) { // create constraint, get property definition for original table // column (maybe could use index column, too) type.setConstraint( new eu.esdihumboldt.hale.common.schema.model.constraint.type.PrimaryKey( Collections.<QName> singletonList(getOrCreateProperty(schema, type, table.getColumn(columns.get(0).getName()), overallNamespace, namespace, types, connection, reporter, catalog) .getName()))); } } // TODO validation of constraints? Most are about several instances (i. // e. UNIQUE) types.addType(type); return type; } /** * @see AbstractIOProvider#getDefaultTypeName() */ @Override protected String getDefaultTypeName() { return "Database"; } /** * Get namespace of the referenced table * * @param referencedTable a {@link Table} * @param overallNamespace the database namespace * @return Namespace of referenced table */ private String getReferencedTableNamespace(Table referencedTable, String overallNamespace) { String namespace; if (overallNamespace.isEmpty()) { namespace = unquote(referencedTable.getSchema().getName()); } else { namespace = overallNamespace; if (referencedTable.getSchema().getName() != null) { namespace += ":" + unquote(referencedTable.getSchema().getName()); } } return namespace; } }