/**
* Licensed to JumpMind Inc under one or more contributor
* license agreements. See the NOTICE file distributed
* with this work for additional information regarding
* copyright ownership. JumpMind Inc licenses this file
* to you under the GNU General Public License, version 3.0 (GPLv3)
* (the "License"); you may not use this file except in compliance
* with the License.
*
* You should have received a copy of the GNU General Public License,
* version 3.0 (GPLv3) along with this library; if not, see
* <http://www.gnu.org/licenses/>.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jumpmind.symmetric.integrate;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.Namespace;
import org.jdom.output.Format;
import org.jdom.output.XMLOutputter;
import org.jumpmind.db.model.Table;
import org.jumpmind.db.platform.IDatabasePlatform;
import org.jumpmind.db.sql.ISqlRowMapper;
import org.jumpmind.db.sql.ISqlTemplate;
import org.jumpmind.db.sql.Row;
import org.jumpmind.db.util.BinaryEncoding;
import org.jumpmind.extension.IExtensionPoint;
import org.jumpmind.symmetric.ISymmetricEngine;
import org.jumpmind.symmetric.ext.INodeGroupExtensionPoint;
import org.jumpmind.symmetric.ext.ISymmetricEngineAware;
import org.jumpmind.symmetric.io.data.Batch;
import org.jumpmind.symmetric.io.data.CsvData;
import org.jumpmind.symmetric.io.data.DataContext;
import org.jumpmind.symmetric.io.data.DataEventType;
import org.jumpmind.util.Context;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.BeanNameAware;
import org.springframework.jmx.export.annotation.ManagedAttribute;
import org.springframework.jmx.export.annotation.ManagedOperation;
import org.springframework.jmx.export.annotation.ManagedOperationParameter;
import org.springframework.jmx.export.annotation.ManagedOperationParameters;
import org.springframework.jmx.export.annotation.ManagedResource;
/**
* An abstract class that accumulates data to publish.
*/
@ManagedResource(description = "The management interface for an xml publisher")
abstract public class AbstractXmlPublisherExtensionPoint implements IExtensionPoint,
INodeGroupExtensionPoint, ISymmetricEngineAware, BeanNameAware {
protected final Logger log = LoggerFactory.getLogger(getClass());
protected final String XML_CACHE = "XML_CACHE_" + this.hashCode();
private String[] nodeGroups;
protected IPublisher publisher;
protected Set<String> tableNamesToPublishAsGroup;
protected String xmlTagNameToUseForGroup = "batch";
protected List<String> groupByColumnNames;
protected Format xmlFormat;
protected String name;
protected ISymmetricEngine engine;
protected long timeBetweenStatisticsPrintTime = 300000;
protected transient long lastStatisticsPrintTime = System.currentTimeMillis();
protected transient long numberOfMessagesPublishedSinceLastPrintTime = 0;
protected transient long amountOfTimeToPublishMessagesSinceLastPrintTime = 0;
protected ITimeGenerator timeStringGenerator = new ITimeGenerator() {
public String getTime() {
return Long.toString(System.currentTimeMillis());
}
};
public AbstractXmlPublisherExtensionPoint() {
xmlFormat = Format.getCompactFormat();
xmlFormat.setOmitDeclaration(true);
}
@Override
public void setBeanName(String name) {
this.name = name;
}
@ManagedOperation(description = "Looks up rows in the database and resends them to the publisher")
@ManagedOperationParameters({ @ManagedOperationParameter(name = "args", description = "A pipe deliminated list of key values to use to look up the tables to resend") })
public boolean resend(String args) {
try {
String[] argArray = args != null ? args.split("\\|") : new String[0];
DataContext context = new DataContext();
IDatabasePlatform platform = engine.getDatabasePlatform();
for (String tableName : tableNamesToPublishAsGroup) {
Table table = platform.getTableFromCache(tableName, false);
List<String[]> dataRowsForTable = readData(table, argArray);
for (String[] values : dataRowsForTable) {
Batch batch = new Batch();
batch.setBinaryEncoding(engine.getSymmetricDialect().getBinaryEncoding());
batch.setSourceNodeId("republish");
context.setBatch(batch);
CsvData data = new CsvData(DataEventType.INSERT);
data.putParsedData(CsvData.ROW_DATA, values);
Element xml = getXmlFromCache(context, context.getBatch().getBinaryEncoding(),
table.getColumnNames(), data.getParsedData(CsvData.ROW_DATA),
table.getPrimaryKeyColumnNames(), data.getParsedData(CsvData.PK_DATA));
if (xml != null) {
toXmlElement(data.getDataEventType(), xml, table.getCatalog(),
table.getSchema(), table.getName(), table.getColumnNames(),
data.getParsedData(CsvData.ROW_DATA),
table.getPrimaryKeyColumnNames(),
data.getParsedData(CsvData.PK_DATA));
}
}
}
if (doesXmlExistToPublish(context)) {
finalizeXmlAndPublish(context);
return true;
} else {
log.warn(String.format(
"Failed to resend message for tables %s, columns %s, and args %s",
tableNamesToPublishAsGroup, groupByColumnNames, args));
}
} catch (RuntimeException ex) {
log.error(String.format(
"Failed to resend message for tables %s, columns %s, and args %s",
tableNamesToPublishAsGroup, groupByColumnNames, args), ex);
}
return false;
}
@ManagedAttribute(description = "A comma separated list of columns that act as the key values for the tables that will be published")
public String getKeyColumnNames() {
return groupByColumnNames != null ? groupByColumnNames.toString() : "";
}
@ManagedAttribute(description = "A comma separated list of tables that will be published")
public String getTableNames() {
return tableNamesToPublishAsGroup != null ? tableNamesToPublishAsGroup.toString() : "";
}
protected List<String[]> readData(final Table table, String[] args) {
final IDatabasePlatform platform = engine.getDatabasePlatform();
List<String[]> rows = new ArrayList<String[]>();
final String[] columnNames = table.getColumnNames();
if (columnNames != null && columnNames.length > 0) {
StringBuilder builder = new StringBuilder("select ");
for (int i = 0; i < columnNames.length; i++) {
String columnName = columnNames[i];
if (i > 0) {
builder.append(",");
}
builder.append(columnName);
}
builder.append(" from ").append(table.getName()).append(" where ");
for (int i = 0; i < groupByColumnNames.size(); i++) {
String columnName = groupByColumnNames.get(i);
if (i > 0 && i < groupByColumnNames.size()) {
builder.append(" and ");
}
builder.append(columnName).append("=?");
}
ISqlTemplate template = platform.getSqlTemplate();
Object[] argObjs = platform.getObjectValues(engine.getSymmetricDialect()
.getBinaryEncoding(), args, table.getColumnsWithName(groupByColumnNames
.toArray(new String[groupByColumnNames.size()])));
rows = template.query(builder.toString(), new ISqlRowMapper<String[]>() {
@Override
public String[] mapRow(Row row) {
return platform.getStringValues(engine.getSymmetricDialect()
.getBinaryEncoding(), table.getColumns(), row, false, false);
}
}, argObjs);
}
return rows;
}
protected final static Namespace getXmlNamespace() {
return Namespace.getNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
}
@SuppressWarnings("unchecked")
protected Map<String, Element> getXmlCache(Context context) {
Map<String, Element> xmlCache = (Map<String, Element>) context.get(XML_CACHE);
if (xmlCache == null) {
xmlCache = new HashMap<String, Element>();
context.put(XML_CACHE, xmlCache);
}
return xmlCache;
}
@SuppressWarnings("unchecked")
protected boolean doesXmlExistToPublish(Context context) {
Map<String, StringBuilder> xmlCache = (Map<String, StringBuilder>) context.get(XML_CACHE);
return xmlCache != null && xmlCache.size() > 0;
}
protected void finalizeXmlAndPublish(Context context) {
Map<String, Element> contextCache = getXmlCache(context);
Collection<Element> buffers = contextCache.values();
for (Iterator<Element> iterator = buffers.iterator(); iterator.hasNext();) {
String xml = new XMLOutputter(xmlFormat).outputString(new Document(iterator.next()));
log.debug("Sending XML to IPublisher: {}", xml);
iterator.remove();
long ts = System.currentTimeMillis();
publisher.publish(context, xml.toString());
amountOfTimeToPublishMessagesSinceLastPrintTime += (System.currentTimeMillis() - ts);
numberOfMessagesPublishedSinceLastPrintTime++;
}
if ((System.currentTimeMillis() - lastStatisticsPrintTime) > timeBetweenStatisticsPrintTime) {
synchronized (this) {
if ((System.currentTimeMillis() - lastStatisticsPrintTime) > timeBetweenStatisticsPrintTime) {
log.info(name
+ " published "
+ numberOfMessagesPublishedSinceLastPrintTime
+ " messages in the last "
+ (System.currentTimeMillis() - lastStatisticsPrintTime)
/ 1000
+ " seconds. Spent "
+ (amountOfTimeToPublishMessagesSinceLastPrintTime / numberOfMessagesPublishedSinceLastPrintTime)
+ "ms of publishing time per message");
lastStatisticsPrintTime = System.currentTimeMillis();
numberOfMessagesPublishedSinceLastPrintTime = 0;
amountOfTimeToPublishMessagesSinceLastPrintTime = 0;
}
}
}
}
protected void toXmlElement(DataEventType dml, Element xml, String catalogName,
String schemaName, String tableName, String[] columnNames, String[] data,
String[] keyNames, String[] keys) {
Element row = new Element("row");
xml.addContent(row);
if (StringUtils.isNotBlank(catalogName)) {
row.setAttribute("catalog", catalogName);
}
if (StringUtils.isNotBlank(schemaName)) {
row.setAttribute("schema", schemaName);
}
row.setAttribute("entity", tableName);
row.setAttribute("dml", dml.getCode());
String[] colNames = null;
if (data == null) {
colNames = keyNames;
data = keys;
} else {
colNames = columnNames;
}
for (int i = 0; i < data.length; i++) {
String col = colNames[i];
Element dataElement = new Element("data");
row.addContent(dataElement);
dataElement.setAttribute("key", col);
if (data[i] != null) {
dataElement.setText(data[i]);
} else {
dataElement.setAttribute("nil", "true", getXmlNamespace());
}
}
}
/**
* Give the opportunity for the user of this publisher to add in additional
* attributes. The default implementation adds in the nodeId from the
* {@link Context}.
*
* @param context
* @param xml
* append XML attributes to this buffer
*/
protected void addFormattedExtraGroupAttributes(Context context, Element xml) {
if (context instanceof DataContext) {
DataContext dataContext = (DataContext) context;
xml.setAttribute("nodeid", dataContext.getBatch().getSourceNodeId());
xml.setAttribute("batchid", Long.toString(dataContext.getBatch().getBatchId()));
}
if (timeStringGenerator != null) {
xml.setAttribute("time", timeStringGenerator.getTime());
}
}
protected Element getXmlFromCache(Context context, BinaryEncoding binaryEncoding,
String[] columnNames, String[] data, String[] keyNames, String[] keys) {
Map<String, Element> xmlCache = getXmlCache(context);
Element xml = null;
String txId = toXmlGroupId(columnNames, data, keyNames, keys);
if (txId != null) {
xml = (Element) xmlCache.get(txId);
if (xml == null) {
xml = new Element(xmlTagNameToUseForGroup);
xml.addNamespaceDeclaration(getXmlNamespace());
xml.setAttribute("id", txId);
xml.setAttribute("binary", binaryEncoding.name());
addFormattedExtraGroupAttributes(context, xml);
xmlCache.put(txId, xml);
}
}
return xml;
}
protected String toXmlGroupId(String[] columnNames, String[] data, String[] keyNames,
String[] keys) {
if (groupByColumnNames != null) {
StringBuilder id = new StringBuilder();
if (keys != null) {
String[] columns = keyNames;
for (String col : groupByColumnNames) {
int index = ArrayUtils.indexOf(columns, col, 0);
if (index >= 0) {
id.append(keys[index]);
} else {
id = new StringBuilder();
break;
}
}
}
if (id.length() == 0) {
String[] columns = columnNames;
for (String col : groupByColumnNames) {
int index = ArrayUtils.indexOf(columns, col, 0);
if (index >= 0) {
id.append(data[index]);
} else {
return null;
}
}
}
if (id.length() > 0) {
return id.toString();
}
} else {
log.warn("You did not specify 'groupByColumnNames'. We cannot find any matches in the data to publish as XML if you don't. You might as well turn off this filter!");
}
return null;
}
public String[] getNodeGroupIdsToApplyTo() {
return nodeGroups;
}
public void setNodeGroups(String[] nodeGroups) {
this.nodeGroups = nodeGroups;
}
public void setNodeGroup(String nodeGroup) {
this.nodeGroups = new String[] { nodeGroup };
}
public void setPublisher(IPublisher publisher) {
this.publisher = publisher;
}
/**
* Used to populate the time attribute of an XML message.
*/
public void setTimeStringGenerator(ITimeGenerator timeStringGenerator) {
this.timeStringGenerator = timeStringGenerator;
}
public void setXmlFormat(Format xmlFormat) {
this.xmlFormat = xmlFormat;
}
public void setTableNamesToPublishAsGroup(Set<String> tableNamesToPublishAsGroup) {
this.tableNamesToPublishAsGroup = tableNamesToPublishAsGroup;
}
public void setTableNameToPublish(String tableName) {
this.tableNamesToPublishAsGroup = new HashSet<String>(1);
this.tableNamesToPublishAsGroup.add(tableName);
}
public void setXmlTagNameToUseForGroup(String xmlTagNameToUseForGroup) {
this.xmlTagNameToUseForGroup = xmlTagNameToUseForGroup;
}
public void setTimeBetweenStatisticsPrintTime(long timeBetweenStatisticsPrintTime) {
this.timeBetweenStatisticsPrintTime = timeBetweenStatisticsPrintTime;
}
/**
* This attribute is required. It needs to identify the columns that will be
* used to key on rows in the specified tables that need to be grouped
* together in an 'XML batch.'
*/
public void setGroupByColumnNames(List<String> groupByColumnNames) {
this.groupByColumnNames = groupByColumnNames;
}
public interface ITimeGenerator {
public String getTime();
}
@Override
public void setSymmetricEngine(ISymmetricEngine engine) {
this.engine = engine;
}
}