MaxwellContext.java

package com.zendesk.maxwell;

import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;

import com.zendesk.maxwell.replication.BinlogPosition;
import com.zendesk.maxwell.bootstrap.AbstractBootstrapper;
import com.zendesk.maxwell.bootstrap.AsynchronousBootstrapper;
import com.zendesk.maxwell.bootstrap.NoOpBootstrapper;
import com.zendesk.maxwell.bootstrap.SynchronousBootstrapper;
import com.zendesk.maxwell.producer.*;
import com.zendesk.maxwell.recovery.RecoveryInfo;
import com.zendesk.maxwell.replication.Position;
import com.zendesk.maxwell.replication.Replicator;
import com.zendesk.maxwell.row.RowMap;
import com.zendesk.maxwell.schema.ReadOnlyMysqlPositionStore;
import com.zendesk.maxwell.schema.MysqlPositionStore;
import com.zendesk.maxwell.schema.PositionStoreThread;

import com.zendesk.maxwell.util.StoppableTask;
import com.zendesk.maxwell.util.TaskManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import snaq.db.ConnectionPool;

public class MaxwellContext {
	static final Logger LOGGER = LoggerFactory.getLogger(MaxwellContext.class);

	private final ConnectionPool replicationConnectionPool;
	private final ConnectionPool maxwellConnectionPool;
	private final ConnectionPool rawMaxwellConnectionPool;
	private final ConnectionPool schemaConnectionPool;
	private final MaxwellConfig config;
	private MysqlPositionStore positionStore;
	private PositionStoreThread positionStoreThread;
	private Long serverID;
	private Position initialPosition;
	private CaseSensitivity caseSensitivity;
	private AbstractProducer producer;
	private TaskManager taskManager;
	private volatile Exception error;

	private Integer mysqlMajorVersion;
	private Integer mysqlMinorVersion;
	private Replicator replicator;

	public MaxwellContext(MaxwellConfig config) throws SQLException {
		this.config = config;
		this.taskManager = new TaskManager();

		this.replicationConnectionPool = new ConnectionPool("ReplicationConnectionPool", 10, 0, 10,
				config.replicationMysql.getConnectionURI(false), config.replicationMysql.user, config.replicationMysql.password);

		if (config.schemaMysql.host == null) {
			this.schemaConnectionPool = null;
		} else {
			this.schemaConnectionPool = new ConnectionPool(
					"SchemaConnectionPool",
					10,
					0,
					10,
					config.schemaMysql.getConnectionURI(false),
					config.schemaMysql.user,
					config.schemaMysql.password);
		}

		this.rawMaxwellConnectionPool = new ConnectionPool("RawMaxwellConnectionPool", 1, 2, 100,
			config.maxwellMysql.getConnectionURI(false), config.maxwellMysql.user, config.maxwellMysql.password);

		this.maxwellConnectionPool = new ConnectionPool("MaxwellConnectionPool", 10, 0, 10,
					config.maxwellMysql.getConnectionURI(), config.maxwellMysql.user, config.maxwellMysql.password);
		this.maxwellConnectionPool.setCaching(false);

		if ( this.config.initPosition != null )
			this.initialPosition = this.config.initPosition;

		if ( this.getConfig().replayMode ) {
			this.positionStore = new ReadOnlyMysqlPositionStore(this.getMaxwellConnectionPool(), this.getServerID(), this.config.clientID, config.gtidMode);
		} else {
			this.positionStore = new MysqlPositionStore(this.getMaxwellConnectionPool(), this.getServerID(), this.config.clientID, config.gtidMode);
		}
	}

	public MaxwellConfig getConfig() {
		return this.config;
	}

	public Connection getReplicationConnection() throws SQLException {
		return this.replicationConnectionPool.getConnection();
	}

	public ConnectionPool getReplicationConnectionPool() { return replicationConnectionPool; }
	public ConnectionPool getMaxwellConnectionPool() { return maxwellConnectionPool; }

	public ConnectionPool getSchemaConnectionPool() {
	    if (this.schemaConnectionPool != null) {
		return schemaConnectionPool;
	    }
	    return replicationConnectionPool;
	}

	public Connection getMaxwellConnection() throws SQLException {
		return this.maxwellConnectionPool.getConnection();
	}

	public Connection getRawMaxwellConnection() throws SQLException {
		return rawMaxwellConnectionPool.getConnection();
	}

	public void start() {
		getPositionStoreThread(); // boot up thread explicitly.
	}

	public void heartbeat() throws Exception {
		this.positionStore.heartbeat();
	}

	public void addTask(StoppableTask task) {
		this.taskManager.add(task);
	}

	public void terminate() {
		terminate(null);
	}

	public void terminate(Exception error) {
		if (this.error == null) {
			this.error = error;
		}

		if (taskManager.requestStop()) {
			if (this.error == null && this.replicator != null) {
				long heartbeat = System.currentTimeMillis();
				LOGGER.info("sending final heartbeat: " + heartbeat);
				try {
					this.replicator.stopAtHeartbeat(heartbeat);
					this.positionStore.heartbeat(heartbeat);
					long deadline = heartbeat + 10000L;
					while (positionStoreThread.getPosition().getLastHeartbeatRead() < heartbeat) {
						if (System.currentTimeMillis() > deadline) {
							LOGGER.warn("timed out waiting for heartbeat " + heartbeat);
							break;
						}
						Thread.sleep(100);
					}
				} catch (Exception e) {
					LOGGER.error("failed graceful shutdown", e);
				}
			}
			taskManager.stop(this.error);
			this.replicationConnectionPool.release();
			this.maxwellConnectionPool.release();
			this.rawMaxwellConnectionPool.release();
		}
	}

	public Exception getError() {
		return error;
	}

	public PositionStoreThread getPositionStoreThread() {
		if ( this.positionStoreThread == null ) {
			this.positionStoreThread = new PositionStoreThread(this.positionStore, this);
			this.positionStoreThread.start();
			addTask(positionStoreThread);
		}
		return this.positionStoreThread;
	}


	public Position getInitialPosition() throws SQLException {
		if ( this.initialPosition != null )
			return this.initialPosition;

		this.initialPosition = this.positionStore.get();
		return this.initialPosition;
	}

	public RecoveryInfo getRecoveryInfo() throws SQLException {
		return this.positionStore.getRecoveryInfo(config);
	}

	public void setPosition(RowMap r) {
		if ( r.isTXCommit() )
			this.setPosition(r.getPosition());
	}

	public void setPosition(Position position) {
		this.getPositionStoreThread().setPosition(position);
	}

	public Position getPosition() throws SQLException {
		return this.getPositionStoreThread().getPosition();
	}

	public MysqlPositionStore getPositionStore() {
		return this.positionStore;
	}

	public Long getServerID() throws SQLException {
		if ( this.serverID != null)
			return this.serverID;

		try ( Connection c = getReplicationConnection() ) {
			ResultSet rs = c.createStatement().executeQuery("SELECT @@server_id as server_id");
			if ( !rs.next() ) {
				throw new RuntimeException("Could not retrieve server_id!");
			}
			this.serverID = rs.getLong("server_id");
			return this.serverID;
		}
	}


	private void fetchMysqlVersion() throws SQLException {
		if ( mysqlMajorVersion == null ) {
			try ( Connection c = getReplicationConnection() ) {
				DatabaseMetaData meta = c.getMetaData();
				mysqlMajorVersion = meta.getDatabaseMajorVersion();
				mysqlMinorVersion = meta.getDatabaseMinorVersion();
			}
		}
	}

	public boolean shouldHeartbeat() throws SQLException {
		fetchMysqlVersion();
		// 5.5 and above
		return (mysqlMajorVersion >= 6) || (mysqlMajorVersion == 5 && mysqlMinorVersion >= 5);
	}

	public CaseSensitivity getCaseSensitivity() throws SQLException {
		if ( this.caseSensitivity != null )
			return this.caseSensitivity;

		try ( Connection c = getReplicationConnection()) {
			ResultSet rs = c.createStatement().executeQuery("select @@lower_case_table_names");
			if ( !rs.next() )
				throw new RuntimeException("Could not retrieve @@lower_case_table_names!");

			int value = rs.getInt(1);
			switch(value) {
				case 0:
					this.caseSensitivity = CaseSensitivity.CASE_SENSITIVE;
					break;
				case 1:
					this.caseSensitivity = CaseSensitivity.CONVERT_TO_LOWER;
					break;
				case 2:
					this.caseSensitivity = CaseSensitivity.CONVERT_ON_COMPARE;
					break;
				default:
					throw new RuntimeException("Unknown value for @@lower_case_table_names: " + value);
			}
			return this.caseSensitivity;
		}
	}

	public AbstractProducer getProducer() throws IOException {
		if ( this.producer != null )
			return this.producer;

		switch ( this.config.producerType ) {
		case "file":
			this.producer = new FileProducer(this, this.config.outputFile);
			break;
		case "kafka":
			this.producer = new MaxwellKafkaProducer(this, this.config.getKafkaProperties(), this.config.kafkaTopic);
			break;
		case "kinesis":
			this.producer = new MaxwellKinesisProducer(this, this.config.kinesisStream);
			break;
		case "profiler":
			this.producer = new ProfilerProducer(this);
			break;
		case "stdout":
			this.producer = new StdoutProducer(this);
			break;
		case "buffer":
			this.producer = new BufferedProducer(this, this.config.bufferedProducerSize);
			break;
		case "none":
			this.producer = null;
			break;
		default:
			throw new RuntimeException("Unknown producer type: " + this.config.producerType);
		}

		StoppableTask task = null;
		if (producer != null) {
			task = producer.getStoppableTask();
		}
		if (task != null) {
			addTask(task);
		}
		return this.producer;
	}

	public AbstractBootstrapper getBootstrapper() throws IOException {
		switch ( this.config.bootstrapperType ) {
			case "async":
				return new AsynchronousBootstrapper(this);
			case "sync":
				return new SynchronousBootstrapper(this);
			default:
				return new NoOpBootstrapper(this);
		}

	}

	public MaxwellFilter getFilter() {
		return config.filter;
	}

	public boolean getReplayMode() {
		return this.config.replayMode;
	}

	private void probePool( ConnectionPool pool, String uri ) throws SQLException {
		try (Connection c = pool.getConnection()) {
			c.createStatement().execute("SELECT 1");
		} catch (SQLException e) {
			LOGGER.error("Could not connect to " + uri + ": " + e.getLocalizedMessage());
			throw (e);
		}
	}

	public void probeConnections() throws SQLException {
		probePool(this.rawMaxwellConnectionPool, this.config.maxwellMysql.getConnectionURI(false));

		if ( this.maxwellConnectionPool != this.replicationConnectionPool )
			probePool(this.replicationConnectionPool, this.config.replicationMysql.getConnectionURI());
	}

	public void setReplicator(Replicator replicator) {
		this.addTask(replicator);
		this.replicator = replicator;
	}
}