/**
* Licensed to JumpMind Inc under one or more contributor
* license agreements. See the NOTICE file distributed
* with this work for additional information regarding
* copyright ownership. JumpMind Inc licenses this file
* to you under the GNU General Public License, version 3.0 (GPLv3)
* (the "License"); you may not use this file except in compliance
* with the License.
*
* You should have received a copy of the GNU General Public License,
* version 3.0 (GPLv3) along with this library; if not, see
* <http://www.gnu.org/licenses/>.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jumpmind.symmetric.io.data.reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang.ArrayUtils;
import org.jumpmind.db.model.Column;
import org.jumpmind.db.model.Table;
import org.jumpmind.db.platform.DatabaseInfo;
import org.jumpmind.db.platform.IDatabasePlatform;
import org.jumpmind.db.sql.ISqlTemplate;
import org.jumpmind.db.util.BinaryEncoding;
import org.jumpmind.symmetric.io.data.Batch;
import org.jumpmind.symmetric.io.data.CsvData;
import org.jumpmind.symmetric.io.data.DataContext;
import org.jumpmind.symmetric.io.data.DataEventType;
import org.jumpmind.symmetric.io.data.IDataReader;
import org.jumpmind.util.CollectionUtils;
import org.jumpmind.util.FormatUtils;
import org.jumpmind.util.Statistics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ExtractDataReader implements IDataReader {
protected static final Logger log = LoggerFactory.getLogger(ExtractDataReader.class);
public static final String DATA_CONTEXT_CURRENT_CSV_DATA = "csvData";
protected Map<Batch, Statistics> statistics = new HashMap<Batch, Statistics>();
protected IDatabasePlatform platform;
protected List<IExtractDataReaderSource> sourcesToUse;
protected IExtractDataReaderSource currentSource;
protected Batch batch;
protected Table table;
protected CsvData data;
protected DataContext dataContext;
public ExtractDataReader(IDatabasePlatform platform, IExtractDataReaderSource source) {
this.sourcesToUse = new ArrayList<IExtractDataReaderSource>();
this.sourcesToUse.add(source);
this.platform = platform;
}
public ExtractDataReader(IDatabasePlatform platform, List<IExtractDataReaderSource> sources) {
this.sourcesToUse = new ArrayList<IExtractDataReaderSource>(sources);
this.platform = platform;
}
public void open(DataContext context) {
this.dataContext = context;
}
public Batch nextBatch() {
closeCurrentSource();
if (this.sourcesToUse.size() > 0) {
this.currentSource = this.sourcesToUse.remove(0);
this.batch = this.currentSource.getBatch();
} else {
this.batch = null;
}
return this.batch;
}
public Table nextTable() {
this.table = null;
if (this.currentSource != null) {
if (this.data == null) {
this.data = this.currentSource.next();
}
if (this.data != null) {
this.table = this.currentSource.getTargetTable();
if (this.table != null) {
this.table.setCatalog(substituteVariables(this.table.getCatalog()));
this.table.setSchema(substituteVariables(this.table.getSchema()));
}
}
}
if (this.table == null && this.batch != null) {
this.batch.setComplete(true);
}
return this.table;
}
protected String substituteVariables(String sourceString) {
if (sourceString != null && sourceString.indexOf("$(") != -1) {
sourceString = FormatUtils.replace("sourceNodeId", (String) dataContext.get("sourceNodeId"), sourceString);
sourceString = FormatUtils.replace("sourceNodeExternalId", (String) dataContext.get("sourceNodeExternalId"), sourceString);
sourceString = FormatUtils.replace("sourceNodeGroupId", (String) dataContext.get("sourceNodeGroupId"), sourceString);
sourceString = FormatUtils.replace("targetNodeId", (String) dataContext.get("targetNodeId"), sourceString);
sourceString = FormatUtils.replace("targetNodeExternalId", (String) dataContext.get("targetNodeExternalId"), sourceString);
sourceString = FormatUtils.replace("targetNodeGroupId", (String) dataContext.get("targetNodeGroupId"), sourceString);
}
return sourceString;
}
public CsvData nextData() {
if (this.table != null) {
if (this.data == null) {
this.data = this.currentSource.next();
}
if (data == null) {
closeCurrentSource();
} else {
Table targetTable = this.currentSource.getTargetTable();
if (targetTable != null && targetTable.equals(this.table)) {
data = enhanceWithLobsFromSourceIfNeeded(this.currentSource.getSourceTable(), data);
} else {
// the table has changed
return null;
}
}
}
CsvData dataToReturn = this.data;
this.data = null;
this.dataContext.put(DATA_CONTEXT_CURRENT_CSV_DATA, dataToReturn);
return dataToReturn;
}
public void close() {
closeCurrentSource();
this.batch = null;
}
protected void closeCurrentSource() {
if (this.currentSource != null) {
this.currentSource.close();
this.currentSource = null;
}
this.table = null;
this.data = null;
}
public Map<Batch, Statistics> getStatistics() {
return statistics;
}
protected CsvData enhanceWithLobsFromSourceIfNeeded(Table table, CsvData data) {
if (this.currentSource.requiresLobsSelectedFromSource()
&& (data.getDataEventType() == DataEventType.UPDATE || data.getDataEventType() == DataEventType.INSERT)) {
List<Column> lobColumns = platform.getLobColumns(table);
if (lobColumns.size() > 0) {
String[] columnNames = table.getColumnNames();
String[] rowData = data.getParsedData(CsvData.ROW_DATA);
Column[] orderedColumns = table.getColumns();
Object[] objectValues = platform.getObjectValues(batch.getBinaryEncoding(),
rowData, orderedColumns);
Map<String, Object> columnDataMap = CollectionUtils
.toMap(columnNames, objectValues);
Column[] pkColumns = table.getPrimaryKeyColumns();
ISqlTemplate sqlTemplate = platform.getSqlTemplate();
Object[] args = new Object[pkColumns.length];
for (int i = 0; i < pkColumns.length; i++) {
args[i] = columnDataMap.get(pkColumns[i].getName());
}
for (Column lobColumn : lobColumns) {
String sql = buildSelect(table, lobColumn, pkColumns);
String valueForCsv = null;
if (platform.isBlob(lobColumn.getMappedTypeCode())) {
byte[] binaryData = sqlTemplate.queryForBlob(sql, lobColumn.getJdbcTypeCode(),lobColumn.getJdbcTypeName(), args);
if (binaryData != null) {
if (batch.getBinaryEncoding() == BinaryEncoding.BASE64) {
valueForCsv = new String(Base64.encodeBase64(binaryData));
} else if (batch.getBinaryEncoding() == BinaryEncoding.HEX) {
valueForCsv = new String(Hex.encodeHex(binaryData));
} else {
valueForCsv = new String(binaryData);
}
binaryData = null;
}
} else {
valueForCsv = sqlTemplate.queryForClob(sql, lobColumn.getJdbcTypeCode(),lobColumn.getJdbcTypeName(), args);
}
int index = ArrayUtils.indexOf(columnNames, lobColumn.getName());
rowData[index] = valueForCsv;
}
data.putParsedData(CsvData.ROW_DATA, rowData);
}
}
return data;
}
protected String buildSelect(Table table, Column lobColumn, Column[] pkColumns) {
StringBuilder sql = new StringBuilder("select ");
DatabaseInfo dbInfo = platform.getDatabaseInfo();
String quote = platform.getDdlBuilder().isDelimitedIdentifierModeOn() ? dbInfo.getDelimiterToken() : "";
sql.append(quote);
sql.append(lobColumn.getName());
sql.append(quote);
sql.append(",");
sql.delete(sql.length() - 1, sql.length());
sql.append(" from ");
sql.append(table.getQualifiedTableName(quote, dbInfo.getCatalogSeparator(),
dbInfo.getSchemaSeparator()));
sql.append(" where ");
for (Column col : pkColumns) {
sql.append(quote);
sql.append(col.getName());
sql.append(quote);
sql.append("=? and ");
}
sql.delete(sql.length() - 5, sql.length());
return sql.toString();
}
}