/**
* Licensed to JumpMind Inc under one or more contributor
* license agreements. See the NOTICE file distributed
* with this work for additional information regarding
* copyright ownership. JumpMind Inc licenses this file
* to you under the GNU General Public License, version 3.0 (GPLv3)
* (the "License"); you may not use this file except in compliance
* with the License.
*
* You should have received a copy of the GNU General Public License,
* version 3.0 (GPLv3) along with this library; if not, see
* <http://www.gnu.org/licenses/>.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jumpmind.symmetric.service.impl;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Map;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.jumpmind.db.platform.AbstractDatabasePlatform;
import org.jumpmind.symmetric.TestConstants;
import org.jumpmind.symmetric.common.ParameterConstants;
import org.jumpmind.symmetric.csv.CsvWriter;
import org.jumpmind.symmetric.ext.NodeGroupTestDataWriterFilter;
import org.jumpmind.symmetric.ext.TestDataWriterFilter;
import org.jumpmind.symmetric.io.data.CsvConstants;
import org.jumpmind.symmetric.io.data.writer.Conflict.ResolveConflict;
import org.jumpmind.symmetric.io.data.writer.IDatabaseWriterFilter;
import org.jumpmind.symmetric.model.IncomingBatch;
import org.jumpmind.symmetric.model.Node;
import org.jumpmind.symmetric.service.IDataLoaderService;
import org.jumpmind.symmetric.service.impl.DataLoaderService.ConflictNodeGroupLink;
import org.jumpmind.symmetric.transport.MockTransportManager;
import org.jumpmind.symmetric.transport.internal.InternalIncomingTransport;
import org.junit.After;
import org.junit.Assert;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
abstract public class AbstractDataLoaderServiceTest extends AbstractServiceTest {
protected final static String TEST_TABLE = "test_dataloader_table";
protected final static String[] TEST_KEYS = { "id" };
protected final static String[] TEST_COLUMNS = { "id", "string_value", "string_required_value",
"char_value", "char_required_value", "date_value", "time_value", "boolean_value",
"integer_value", "decimal_value", "double_value" };
protected static int batchId = 10000;
protected static int sequenceId = 10000;
protected Node client = new Node(TestConstants.TEST_CLIENT_EXTERNAL_ID, null, null);
protected Node root = new Node(TestConstants.TEST_ROOT_EXTERNAL_ID, null, null);
private MockTransportManager transportManager;
protected synchronized String getNextBatchId() {
return Integer.toString(++batchId);
}
protected synchronized String getBatchId() {
return Integer.toString(batchId);
}
protected synchronized String getNextId() {
return Integer.toString(++sequenceId);
}
protected synchronized String getId() {
return Integer.toString(sequenceId);
}
@Test
public void test01IncomingBatch() throws Exception {
String[] insertValues = new String[TEST_COLUMNS.length];
insertValues[2] = insertValues[4] = "incoming test";
ByteArrayOutputStream out = new ByteArrayOutputStream();
CsvWriter writer = getWriter(out);
writer.writeRecord(new String[] { CsvConstants.NODEID,
TestConstants.TEST_CLIENT_EXTERNAL_ID });
writer.writeRecord(new String[] { CsvConstants.CHANNEL,
TestConstants.TEST_CHANNEL_ID });
String nextBatchId = getNextBatchId();
writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
insertValues[0] = getNextId();
writer.write(CsvConstants.INSERT);
writer.writeRecord(insertValues, true);
writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
writer.close();
load(out);
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status. " + printDatabase());
assertEquals(batch.getChannelId(), TestConstants.TEST_CHANNEL_ID, "Wrong channel. " + printDatabase());
}
@Test
public void test02Statistics() throws Exception {
Level old = setLoggingLevelForTest(Level.FATAL);
String[] updateValues = new String[TEST_COLUMNS.length + 1];
updateValues[0] = updateValues[updateValues.length - 1] = getNextId();
updateValues[2] = updateValues[4] = "required string";
String[] insertValues = (String[]) ArrayUtils.subarray(updateValues, 0,
updateValues.length - 1);
ByteArrayOutputStream out = new ByteArrayOutputStream();
CsvWriter writer = getWriter(out);
writer.writeRecord(new String[] { CsvConstants.NODEID,
TestConstants.TEST_CLIENT_EXTERNAL_ID });
writer.writeRecord(new String[] { CsvConstants.CHANNEL,
TestConstants.TEST_CHANNEL_ID });
String nextBatchId = getNextBatchId();
writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
// Update becomes fallback insert
writer.write(CsvConstants.UPDATE);
writer.writeRecord(updateValues, true);
// Insert becomes fallback update
writer.write(CsvConstants.INSERT);
writer.writeRecord(insertValues, true);
// Insert becomes fallback update
writer.write(CsvConstants.INSERT);
writer.writeRecord(insertValues, true);
// Clean insert
insertValues[0] = getNextId();
writer.write(CsvConstants.INSERT);
writer.writeRecord(insertValues, true);
// Delete affects no rows
writer.writeRecord(new String[] { CsvConstants.DELETE, getNextId() }, true);
writer.writeRecord(new String[] { CsvConstants.DELETE, getNextId() }, true);
writer.writeRecord(new String[] { CsvConstants.DELETE, getNextId() }, true);
// Failing statement
insertValues[0] = getNextId();
insertValues[5] = "i am an invalid date";
writer.write(CsvConstants.INSERT);
writer.writeRecord(insertValues, true);
writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
writer.close();
load(out);
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNotNull(batch);
assertEquals(batch.getStatus(), IncomingBatch.Status.ER, "Wrong status. " + printDatabase());
assertEquals(batch.getFailedRowNumber(), 8l, "Wrong failed row number. " + batch.getSqlMessage() + ". " + printDatabase());
assertEquals(batch.getByteCount(), 496l, "Wrong byte count. " + printDatabase());
assertEquals(batch.getStatementCount(), 8l, "Wrong statement count. " + printDatabase());
assertEquals(batch.getFallbackInsertCount(), 1l, "Wrong fallback insert count. "
+ printDatabase());
assertEquals(batch.getFallbackUpdateCount(), 2l, "Wrong fallback update count. "
+ printDatabase());
assertEquals(batch.getMissingDeleteCount(), 3l, "Wrong missing delete count. "
+ printDatabase());
setLoggingLevelForTest(old);
}
@Test
public void test03UpdateCollision() throws Exception {
Level old = setLoggingLevelForTest(Level.OFF);
String[] insertValues = new String[TEST_COLUMNS.length];
insertValues[0] = getNextId();
insertValues[2] = insertValues[4] = "inserted row for testUpdateCollision";
String[] updateValues = new String[TEST_COLUMNS.length];
updateValues[0] = getId();
updateValues[TEST_COLUMNS.length - 1] = getNextId();
updateValues[2] = updateValues[4] = "update will become an insert that violates PK";
ByteArrayOutputStream out = new ByteArrayOutputStream();
CsvWriter writer = getWriter(out);
writer.writeRecord(new String[] { CsvConstants.NODEID,
TestConstants.TEST_CLIENT_EXTERNAL_ID });
writer.writeRecord(new String[] { CsvConstants.CHANNEL,
TestConstants.TEST_CHANNEL_ID });
String nextBatchId = getNextBatchId();
writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
// This insert will be OK
writer.write(CsvConstants.INSERT);
writer.writeRecord(insertValues, true);
// Update becomes fallback insert, and then violate the primary key
writer.write(CsvConstants.UPDATE);
writer.writeRecord(updateValues, true);
writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
writer.close();
load(out);
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNotNull(batch);
load(out);
assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status");
batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNotNull(batch);
assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status");
setLoggingLevelForTest(old);
}
@Test
public void test04SqlStatistics() throws Exception {
Level old = setLoggingLevelForTest(Level.OFF);
String[] insertValues = new String[TEST_COLUMNS.length];
insertValues[2] = insertValues[4] = "sql stat test";
ByteArrayOutputStream out = new ByteArrayOutputStream();
CsvWriter writer = getWriter(out);
writer.writeRecord(new String[] { CsvConstants.NODEID,
TestConstants.TEST_CLIENT_EXTERNAL_ID });
writer.writeRecord(new String[] { CsvConstants.CHANNEL,
TestConstants.TEST_CHANNEL_ID });
String nextBatchId = getNextBatchId();
writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
// Clean insert
String firstId = getNextId();
insertValues[0] = firstId;
writer.write(CsvConstants.INSERT);
writer.writeRecord(insertValues, true);
// Clean insert
String secondId = getNextId();
insertValues[0] = secondId;
writer.write(CsvConstants.INSERT);
writer.writeRecord(insertValues, true);
String thirdId = getNextId();
insertValues[0] = thirdId;
// date column ...
insertValues[5] = "This is a very long string that will fail upon insert into the database.";
writer.write(CsvConstants.INSERT);
writer.writeRecord(insertValues, true);
writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
writer.close();
load(out);
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNotNull(batch);
assertEquals(batch.getStatus(), IncomingBatch.Status.ER, "Wrong status. " + printDatabase());
assertEquals(batch.getFailedRowNumber(), 3l, "Wrong failed row number. " + printDatabase());
Assert.assertEquals("Wrong byte count: " + batch.getByteCount() + ". " + printDatabase(), 407,
batch.getByteCount());
assertEquals(batch.getStatementCount(), 3l, "Wrong statement count. " + printDatabase());
assertEquals(batch.getFallbackInsertCount(), 0l, "Wrong fallback insert count. "
+ printDatabase());
assertEquals(batch.getFallbackUpdateCount(), 0l, "Wrong fallback update count. "
+ printDatabase());
assertEquals(batch.getMissingDeleteCount(), 0l, "Wrong missing delete count. "
+ printDatabase());
assertNull(batch.getSqlState(), "Sql state should be null. " + printDatabase());
assertNotNull(batch.getSqlMessage(), "Sql message should not be null. " + printDatabase());
setLoggingLevelForTest(old);
}
@Test
public void test05SkippingResentBatch() throws Exception {
String[] values = { getNextId(), "resend string", "resend string not null", "resend char",
"resend char not null", "2007-01-25 00:00:00.000", "2007-01-25 01:01:01.000", "0", "7",
"10.10", "0.474" };
getNextBatchId();
for (long i = 0; i < 7; i++) {
batchId--;
testSimple(CsvConstants.INSERT, values, values);
assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID),
IncomingBatch.Status.OK, "Wrong status");
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNotNull(batch);
assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status");
assertEquals(batch.getSkipCount(), i);
assertEquals(batch.getFailedRowNumber(), 0l, "Wrong failed row number");
assertEquals(batch.getStatementCount(), 1l, "Wrong statement count");
assertEquals(batch.getFallbackInsertCount(), 0l, "Wrong fallback insert count");
assertEquals(batch.getFallbackUpdateCount(), 0l, "Wrong fallback update count");
// pause to make sure we get a different start time on the incoming
// batch batch
Thread.sleep(10);
}
}
@Test
public void test06ErrorWhileSkip() throws Exception {
Level old = setLoggingLevelForTest(Level.OFF);
String[] values = { getNextId(), "string2", "string not null2", "char2", "char not null2",
"2007-01-02 00:00:00.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "0.474" };
testSimple(CsvConstants.INSERT, values, values);
assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID),
IncomingBatch.Status.OK, "Wrong status");
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNotNull(batch);
assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status");
assertEquals(batch.getFailedRowNumber(), 0l, "Wrong failed row number");
assertEquals(batch.getStatementCount(), 1l, "Wrong statement count");
ByteArrayOutputStream out = new ByteArrayOutputStream();
CsvWriter writer = getWriter(out);
writer.writeRecord(new String[] { CsvConstants.NODEID,
TestConstants.TEST_CLIENT_EXTERNAL_ID });
writer.writeRecord(new String[] { CsvConstants.CHANNEL,
TestConstants.TEST_CHANNEL_ID });
writer.writeRecord(new String[] { CsvConstants.BATCH, getBatchId() });
writer.write(CsvConstants.KEYS);
writer.writeRecord(TEST_KEYS);
writer.writeRecord(new String[] { CsvConstants.COMMIT, getBatchId() });
writer.close();
// Pause a moment to guarantee our batch comes back in time order
Thread.sleep(10);
load(out);
assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID),
IncomingBatch.Status.OK, "Wrong status");
setLoggingLevelForTest(old);
}
@Test
public void test07DataIntregrityError() throws Exception {
Level old = setLoggingLevelForTest(Level.OFF);
String[] values = { getNextId(), "string3", "string not null3", "char3", "char not null3",
"2007-01-02 00:00:00.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "0.474" };
ConflictNodeGroupLink conflictSettings = new ConflictNodeGroupLink();
conflictSettings.setNodeGroupLink(TestConstants.TEST_2_ROOT);
conflictSettings.setConflictId("dont_fallback");
conflictSettings.setResolveType(ResolveConflict.MANUAL);
getSymmetricEngine().getDataLoaderService().save(conflictSettings);
ByteArrayOutputStream out = new ByteArrayOutputStream();
CsvWriter writer = getWriter(out);
writer.writeRecord(new String[] { CsvConstants.NODEID,
TestConstants.TEST_CLIENT_EXTERNAL_ID });
writer.writeRecord(new String[] { CsvConstants.CHANNEL,
TestConstants.TEST_CHANNEL_ID });
writer.writeRecord(new String[] { CsvConstants.BATCH, getNextBatchId() });
writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
writer.write(CsvConstants.INSERT);
writer.writeRecord(values, true);
writer.write(CsvConstants.INSERT);
writer.writeRecord(values, true);
writer.writeRecord(new String[] { CsvConstants.COMMIT, getBatchId() });
writer.close();
load(out);
assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID),
IncomingBatch.Status.ER, "Wrong status");
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNotNull(batch);
assertEquals(batch.getStatus(), IncomingBatch.Status.ER, "Wrong status");
assertEquals(batch.getFailedRowNumber(), 2l, "Wrong failed row number");
assertEquals(batch.getStatementCount(), 2l, "Wrong statement count");
load(out);
assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID),
IncomingBatch.Status.ER, "Wrong status");
batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNotNull(batch);
assertEquals(batch.getStatus(), IncomingBatch.Status.ER, "Wrong status");
assertEquals(batch.getFailedRowNumber(), 2l, "Wrong failed row number");
assertEquals(batch.getStatementCount(), 2l, "Wrong statement count");
getSymmetricEngine().getDataLoaderService().delete(conflictSettings);
setLoggingLevelForTest(old);
}
@Test
public void test08ErrorWhileParsing() throws Exception {
Level old = setLoggingLevelForTest(Level.OFF);
String[] values = { getNextId(), "should not reach database", "string not null", "char",
"char not null", "2007-01-02", "2007-02-03 04:05:06.000", "0", "47", "67.89", "0.474" };
ByteArrayOutputStream out = new ByteArrayOutputStream();
CsvWriter writer = getWriter(out);
writer.writeRecord(new String[] { CsvConstants.NODEID,
TestConstants.TEST_CLIENT_EXTERNAL_ID });
writer.writeRecord(new String[] { CsvConstants.CHANNEL,
TestConstants.TEST_CHANNEL_ID });
String nextBatchId = getNextBatchId();
writer.write("UnknownTokenOutsideBatch");
writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
writer.writeRecord(new String[] { CsvConstants.TABLE, TEST_TABLE });
writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
writer.write(CsvConstants.INSERT);
writer.writeRecord(values, true);
writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
writer.close();
load(out);
assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID), null,
"Wrong status");
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNull(batch);
setLoggingLevelForTest(old);
}
@Test
public void test09ErrorThenSuccessBatch() throws Exception {
Logger.getLogger(AbstractDataLoaderServiceTest.class).warn("testErrorThenSuccessBatch");
Level old = setLoggingLevelForTest(Level.OFF);
String[] values = { getNextId(),
"This string is too large and will cause the statement to fail",
"string not null2", "char2", "char not null2", "not a date",
"2007-02-03 04:05:06.000", "0", "47", "123456789.00", "0.474" };
getNextBatchId();
int retries = 3;
for (int i = 0; i < retries; i++) {
batchId--;
testSimple(CsvConstants.INSERT, values, null);
assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID),
IncomingBatch.Status.ER, "Wrong status");
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNotNull(batch);
assertEquals(batch.getStatus(), IncomingBatch.Status.ER, "Wrong status. "
+ printDatabase());
assertEquals(batch.getFailedRowNumber(), 1l, "Wrong failed row number. "
+ printDatabase());
assertEquals(batch.getStatementCount(), 1l, "Wrong statement count. " + printDatabase());
// pause to make sure we get a different start time on the incoming
// batch batch
Thread.sleep(10);
}
batchId--;
values[1] = "A smaller string that will succeed";
values[5] = "2007-01-02 00:00:00.000";
values[9] = "67.89";
testSimple(CsvConstants.INSERT, values, values);
assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID),
IncomingBatch.Status.OK, "Wrong status. " + printDatabase());
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId,
TestConstants.TEST_CLIENT_EXTERNAL_ID);
assertNotNull(batch);
assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status. " + printDatabase());
assertEquals(batch.getFailedRowNumber(), 0l, "Wrong failed row number. " + printDatabase());
assertEquals(batch.getStatementCount(), 1l, "Wrong statement count. " + printDatabase());
setLoggingLevelForTest(old);
}
@Test
public void test10MultipleBatch() throws Exception {
Level old = setLoggingLevelForTest(Level.OFF);
String[] values = { getNextId(), "string", "string not null2", "char2", "char not null2",
"2007-01-02 00:00:00.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "0.474" };
String[] values2 = { getNextId(),
"This string is too large and will cause the statement to fail",
"string not null2", "char2", "char not null2", "Not a date",
"2007-02-03 04:05:06.000", "0", "47", "123456789.00", "0.474" };
ByteArrayOutputStream out = new ByteArrayOutputStream();
CsvWriter writer = getWriter(out);
writer.writeRecord(new String[] { CsvConstants.NODEID,
TestConstants.TEST_CLIENT_EXTERNAL_ID });
writer.writeRecord(new String[] { CsvConstants.CHANNEL,
TestConstants.TEST_CHANNEL_ID });
String nextBatchId = getNextBatchId();
writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
writer.write(CsvConstants.INSERT);
writer.writeRecord(values, true);
writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
String nextBatchId2 = getNextBatchId();
writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId2 });
writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
writer.write(CsvConstants.INSERT);
writer.writeRecord(values2, true);
writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId2 });
writer.close();
load(out);
assertTestTableEquals(values[0], values);
assertTestTableEquals(values2[0], null);
assertEquals(
findIncomingBatchStatus(Integer.parseInt(nextBatchId),
TestConstants.TEST_CLIENT_EXTERNAL_ID), IncomingBatch.Status.OK,
"Wrong status. " + printDatabase());
assertEquals(
findIncomingBatchStatus(Integer.parseInt(nextBatchId2),
TestConstants.TEST_CLIENT_EXTERNAL_ID), IncomingBatch.Status.ER,
"Wrong status. " + printDatabase());
setLoggingLevelForTest(old);
}
protected void testSimple(String dmlType, String[] values, String[] expectedValues)
throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
CsvWriter writer = getWriter(out);
writer.writeRecord(new String[] { CsvConstants.NODEID,
TestConstants.TEST_CLIENT_EXTERNAL_ID });
writer.writeRecord(new String[] { CsvConstants.CHANNEL,
TestConstants.TEST_CHANNEL_ID });
String nextBatchId = getNextBatchId();
writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
writer.write(dmlType);
writer.writeRecord(values, true);
writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
writer.close();
load(out);
assertTestTableEquals(values[0], expectedValues);
}
protected void load(ByteArrayOutputStream out) throws Exception {
ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
getTransportManager().setIncomingTransport(new InternalIncomingTransport(in));
getDataLoaderService().loadDataFromPull(client);
}
protected IncomingBatch.Status findIncomingBatchStatus(int batchId, String nodeId) {
IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId, nodeId);
IncomingBatch.Status status = null;
if (batch != null) {
status = batch.getStatus();
}
return status;
}
@Test
public void test12RegisteredDataWriterFilter() {
TestDataWriterFilter registeredFilter = (TestDataWriterFilter) getSymmetricEngine().getExtensionService()
.getExtensionPointMap(IDatabaseWriterFilter.class).get("registeredDataFilter");
assertTrue(registeredFilter.getNumberOfTimesCalled() > 0);
NodeGroupTestDataWriterFilter registeredNodeGroupFilter = (NodeGroupTestDataWriterFilter) getSymmetricEngine()
.getExtensionService().getExtensionPointMap(IDatabaseWriterFilter.class).get("registeredNodeGroupTestDataFilter");
assertTrue(registeredNodeGroupFilter.getNumberOfTimesCalled() > 0);
}
protected CsvWriter getWriter(OutputStream out) {
CsvWriter writer = new CsvWriter(new OutputStreamWriter(out), ',');
writer.setEscapeMode(CsvWriter.ESCAPE_MODE_BACKSLASH);
return writer;
}
protected MockTransportManager getTransportManager() {
if (transportManager == null) {
transportManager = new MockTransportManager();
}
return transportManager;
}
protected void writeTable(CsvWriter writer, String tableName, String[] keys, String[] columns)
throws IOException {
writer.writeRecord(new String[] { "table", tableName });
writer.write("keys");
writer.writeRecord(keys);
writer.write("columns");
writer.writeRecord(columns);
}
protected void assertTestTableEquals(String testTableId, String[] expectedValues) {
String sql = "select " + getSelect(TEST_COLUMNS) + " from " + TEST_TABLE + " where "
+ getWhere(TEST_KEYS);
Map<String, Object> results = getSqlTemplate().queryForMap(sql, new Object[] { new Long(testTableId) });
if (expectedValues != null) {
expectedValues[1] = translateExpectedString(expectedValues[1], false);
expectedValues[2] = translateExpectedString(expectedValues[2], true);
expectedValues[3] = translateExpectedCharString(expectedValues[3], 50, false);
expectedValues[4] = translateExpectedCharString(expectedValues[4], 50, true);
}
assertEquals(TEST_COLUMNS, expectedValues, results);
}
protected void assertEquals(String[] name, String[] expected, Map<String, Object> results) {
if (expected == null) {
Assert.assertNull("Expected empty results. " + printDatabase(), results);
} else {
Assert.assertNotNull("Expected non-empty results. " + printDatabase(), results);
for (int i = 0; i < expected.length; i++) {
Object resultObj = results.get(name[i]);
String resultValue = null;
char decimal = ((DecimalFormat) DecimalFormat.getInstance())
.getDecimalFormatSymbols().getDecimalSeparator();
if ((resultObj instanceof Double || resultObj instanceof BigDecimal) && expected[i].indexOf(decimal) != -1) {
DecimalFormat df = new DecimalFormat("0.00####################################");
resultValue = df.format(resultObj);
} else if (resultObj instanceof Date) {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.000");
resultValue = df.format(resultObj);
} else if (resultObj instanceof Boolean) {
resultValue = ((Boolean) resultObj) ? "1" : "0";
} else if (resultObj instanceof Double) {
resultValue = resultObj.toString();
} else if (resultObj != null) {
resultValue = resultObj.toString();
}
Assert.assertEquals(name[i] + ". " + printDatabase(), expected[i], resultValue);
}
}
}
protected String getSelect(String[] columns) {
StringBuilder str = new StringBuilder();
for (int i = 0; i < columns.length; i++) {
str.append(columns[i]).append(i + 1 < columns.length ? ", " : "");
}
return str.toString();
}
protected String getWhere(String[] columns) {
StringBuilder str = new StringBuilder();
for (int i = 0; i < columns.length; i++) {
str.append(columns[i]).append(" = ?").append(i + 1 < columns.length ? "," : "");
}
return str.toString();
}
protected String translateExpectedString(String value, boolean isRequired) {
if (isRequired
&& (value == null || (value.equals("") && getDbDialect().getPlatform()
.getDatabaseInfo().isEmptyStringNulled()))) {
return AbstractDatabasePlatform.REQUIRED_FIELD_NULL_SUBSTITUTE;
} else if (value != null && value.equals("")
&& getDbDialect().getPlatform().getDatabaseInfo().isEmptyStringNulled()) {
return null;
}
return value;
}
protected String translateExpectedCharString(String value, int size, boolean isRequired) {
if (isRequired && value == null) {
value = AbstractDatabasePlatform.REQUIRED_FIELD_NULL_SUBSTITUTE;
}
if (value != null
&& ((StringUtils.isBlank(value) && getDbDialect().getPlatform().getDatabaseInfo()
.isBlankCharColumnSpacePadded()) || (StringUtils.isNotBlank(value) && getDbDialect()
.getPlatform().getDatabaseInfo().isNonBlankCharColumnSpacePadded()))) {
return StringUtils.rightPad(value, size);
} else if (value != null
&& getDbDialect().getPlatform().getDatabaseInfo().isCharColumnSpaceTrimmed()) {
return value.replaceFirst(" *$", "");
}
return value;
}
protected IDataLoaderService getDataLoaderService() {
DataLoaderService dataLoaderService = (DataLoaderService) getSymmetricEngine()
.getDataLoaderService();
dataLoaderService.setTransportManager(transportManager);
return dataLoaderService;
}
@After
public void cleanup() {
getSymmetricEngine().getStagingManager().clean(getSymmetricEngine().getParameterService()
.getLong(ParameterConstants.STREAM_TO_FILE_TIME_TO_LIVE_MS));
}
}