package com.zendesk.maxwell.recovery;
import com.zendesk.maxwell.*;
import com.zendesk.maxwell.replication.Position;
import com.zendesk.maxwell.row.RowMap;
import com.zendesk.maxwell.schema.MysqlSavedSchema;
import com.zendesk.maxwell.schema.Schema;
import com.zendesk.maxwell.schema.SchemaCapturer;
import com.zendesk.maxwell.schema.SchemaStoreSchema;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
public class RecoveryTest extends TestWithNameLogging {
private static MysqlIsolatedServer masterServer, slaveServer;
static final Logger LOGGER = LoggerFactory.getLogger(RecoveryTest.class);
private static final int DATA_SIZE = 500;
private static final int NEW_DATA_SIZE = 100;
@Before
public void setupServers() throws Exception {
masterServer = new MysqlIsolatedServer();
masterServer.boot();
SchemaStoreSchema.ensureMaxwellSchema(masterServer.getConnection(), "maxwell");
slaveServer = MaxwellTestSupport.setupServer("--server_id=12345 --max_binlog_size=100000 --log_bin=slave");
slaveServer.setupSlave(masterServer.getPort());
MaxwellTestSupport.setupSchema(masterServer, false);
}
private MaxwellConfig getConfig(int port, boolean masterRecovery) {
MaxwellConfig config = new MaxwellConfig();
config.maxwellMysql.host = "localhost";
config.maxwellMysql.port = port;
config.maxwellMysql.user = "maxwell";
config.maxwellMysql.password = "maxwell";
config.masterRecovery = masterRecovery;
config.maxwellMysql.jdbcOptions.add("useSSL=false");
config.validate();
return config;
}
private MaxwellContext getContext(int port, boolean masterRecovery) throws SQLException {
MaxwellConfig config = getConfig(port, masterRecovery);
return new MaxwellContext(config);
}
private String[] generateMasterData() throws Exception {
String input[] = new String[DATA_SIZE];
for ( int i = 0 ; i < DATA_SIZE; i++ ) {
input[i] = String.format("insert into shard_1.minimal set account_id = %d, text_field='row %d'", i, i);
}
return input;
}
private void generateNewMasterData(boolean useMaster, int startNum) throws Exception {
MysqlIsolatedServer server = useMaster ? masterServer : slaveServer;
for ( int i = 0 ; i < NEW_DATA_SIZE; i++ ) {
server.execute(String.format("insert into shard_1.minimal set account_id = %d, text_field='row %d'", i + startNum, i + startNum));
if ( i % 100 == 0 )
server.execute("flush logs");
}
}
@Test
public void testBasicRecovery() throws Exception {
if (MaxwellTestSupport.inGtidMode()) {
LOGGER.info("No need to test recovery under gtid-mode");
return;
}
MaxwellContext slaveContext = getContext(slaveServer.getPort(), true);
String[] input = generateMasterData();
/* run the execution through with the replicator running so we get heartbeats */
MaxwellTestSupport.getRowsWithReplicator(masterServer, null, input, null);
Position slavePosition = MaxwellTestSupport.capture(slaveServer.getConnection());
generateNewMasterData(false, DATA_SIZE);
RecoveryInfo recoveryInfo = slaveContext.getRecoveryInfo();
assertThat(recoveryInfo, notNullValue());
MaxwellConfig slaveConfig = getConfig(slaveServer.getPort(), true);
Recovery recovery = new Recovery(
slaveConfig.maxwellMysql,
slaveConfig.databaseName,
slaveContext.getReplicationConnectionPool(),
slaveContext.getCaseSensitivity(),
recoveryInfo,
System.getenv("SHYKO_MODE") != null
);
Position recoveredPosition = recovery.recover();
// lousy tests, but it's very hard to make firm assertions about the correct position.
// It's in a ballpark.
if ( slavePosition.getBinlogPosition().getFile().equals(recoveredPosition.getBinlogPosition().getFile()) ) {
long positionDiff = recoveredPosition.getBinlogPosition().getOffset() - slavePosition.getBinlogPosition().getOffset();
assertThat(Math.abs(positionDiff), lessThan(1500L));
} else {
// TODO: something something.
}
}
@Test
public void testOtherClientID() throws Exception {
if (MaxwellTestSupport.inGtidMode()) {
LOGGER.info("No need to test recovery under gtid-mode");
return;
}
MaxwellContext slaveContext = getContext(slaveServer.getPort(), true);
String[] input = generateMasterData();
MaxwellTestSupport.getRowsWithReplicator(masterServer, null, input, null);
generateNewMasterData(false, DATA_SIZE);
RecoveryInfo recoveryInfo = slaveContext.getRecoveryInfo();
assertThat(recoveryInfo, notNullValue());
/* pretend that we're a seperate client trying to recover now */
recoveryInfo.clientID = "another_client";
MaxwellConfig slaveConfig = getConfig(slaveServer.getPort(), true);
Recovery recovery = new Recovery(
slaveConfig.maxwellMysql,
slaveConfig.databaseName,
slaveContext.getReplicationConnectionPool(),
slaveContext.getCaseSensitivity(),
recoveryInfo,
System.getenv("SHYKO_MODE") != null
);
Position recoveredPosition = recovery.recover();
assertEquals(null, recoveredPosition);
}
/* i know. it's horrible. */
private void drainReplication(BufferedMaxwell maxwell, List<RowMap> rows) throws IOException, InterruptedException {
int pollMS = 10000;
for ( ;; ) {
RowMap r = maxwell.poll(pollMS);
if ( r == null )
break;
else {
if ( r.toJSON() != null )
rows.add(r);
pollMS = 500; // once we get a row, we timeout quickly.
}
}
}
@Test
public void testRecoveryIntegration() throws Exception {
if (MaxwellTestSupport.inGtidMode()) {
LOGGER.info("No need to test recovery under gtid-mode");
return;
}
String[] input = generateMasterData();
/* run the execution through with the replicator running so we get heartbeats */
List<RowMap> rows = MaxwellTestSupport.getRowsWithReplicator(masterServer, null, input, null);
Position approximateRecoverPosition = MaxwellTestSupport.capture(slaveServer.getConnection());
LOGGER.warn("slave master position at time of cut: " + approximateRecoverPosition);
generateNewMasterData(false, DATA_SIZE);
BufferedMaxwell maxwell = new BufferedMaxwell(getConfig(slaveServer.getPort(), true));
new Thread(maxwell).start();
drainReplication(maxwell, rows);
for ( long i = 0 ; i < DATA_SIZE + NEW_DATA_SIZE; i++ ) {
assertEquals(i + 1, rows.get((int) i).getData("id"));
}
// assert that we created a schema that matches up with the matched position.
ResultSet rs = slaveServer.getConnection().createStatement().executeQuery("select * from maxwell.schemas");
boolean foundSchema = false;
while ( rs.next() ) {
if ( rs.getLong("server_id") == 12345 ) {
foundSchema = true;
rs.getLong("base_schema_id");
assertEquals(false, rs.wasNull());
}
}
assertEquals(true, foundSchema);
maxwell.terminate();
// assert that we deleted the old position row
rs = slaveServer.getConnection().createStatement().executeQuery("select * from maxwell.positions");
rs.next();
assertEquals(12345, rs.getLong("server_id"));
assert(!rs.next());
}
@Test
public void testRecoveryIntegrationWithLaggedMaxwell() throws Exception {
if (MaxwellTestSupport.inGtidMode()) {
LOGGER.info("No need to test recovery under gtid-mode");
return;
}
final String[] input = generateMasterData();
MaxwellTestSupportCallback callback = new MaxwellTestSupportCallback() {
@Override
public void afterReplicatorStart(MysqlIsolatedServer mysql) throws SQLException {
mysql.executeList(Arrays.asList(input));
}
@Override
public void beforeTerminate(MysqlIsolatedServer mysql) {
/* record some queries. maxwell may continue to heartbeat but we will be behind. */
try {
LOGGER.warn("slave master position at time of cut: " + MaxwellTestSupport.capture(slaveServer.getConnection()));
mysql.executeList(Arrays.asList(input));
mysql.execute("FLUSH LOGS");
mysql.executeList(Arrays.asList(input));
mysql.execute("FLUSH LOGS");
} catch ( Exception e ) {}
}
};
List<RowMap> rows = MaxwellTestSupport.getRowsWithReplicator(masterServer, null, callback, null);
generateNewMasterData(false, DATA_SIZE);
BufferedMaxwell maxwell = new BufferedMaxwell(getConfig(slaveServer.getPort(), true));
new Thread(maxwell).start();
drainReplication(maxwell, rows);
assertThat(rows.size(), greaterThanOrEqualTo(1600));
boolean[] ids = new boolean[1601];
for ( RowMap r : rows ) {
Long id = (Long) r.getData("id");
if ( id != null )
ids[id.intValue()] = true;
}
for ( int i = 1 ; i < 1601; i++ )
assertEquals("didn't find id " + i, true, ids[i]);
maxwell.terminate();
}
@Test
public void testFailOver() throws Exception {
String[] input = generateMasterData();
// Have maxwell connect to master first
List<RowMap> rows = MaxwellTestSupport.getRowsWithReplicator(masterServer, null, input, null);
try {
// sleep a bit for slave to catch up
Thread.sleep(1000);
} catch (InterruptedException ex) {
LOGGER.info("Got ex: " + ex);
}
Position slavePosition1 = MaxwellTestSupport.capture(slaveServer.getConnection());
LOGGER.info("slave master position at time of cut: " + slavePosition1 + " rows: " + rows.size());
// add 1000 rows on master side
generateNewMasterData(true, DATA_SIZE);
// connect to slave, maxwell should get these 1000 rows from slave
boolean masterRecovery = !MaxwellTestSupport.inGtidMode();
BufferedMaxwell maxwell = new BufferedMaxwell(getConfig(slaveServer.getPort(), masterRecovery));
new Thread(maxwell).start();
drainReplication(maxwell, rows);
maxwell.terminate();
Position slavePosition2 = MaxwellTestSupport.capture(slaveServer.getConnection());
LOGGER.info("slave master position after failover: " + slavePosition2 + " rows: " + rows.size());
assertTrue(slavePosition2.newerThan(slavePosition1));
// add 1000 rows on slave side
generateNewMasterData(false, DATA_SIZE + NEW_DATA_SIZE);
// reconnct to slave to resume, maxwell should get the new 1000 rows
maxwell = new BufferedMaxwell(getConfig(slaveServer.getPort(), false));
new Thread(maxwell).start();
drainReplication(maxwell, rows);
maxwell.terminate();
Position slavePosition3 = MaxwellTestSupport.capture(slaveServer.getConnection());
LOGGER.info("slave master position after resumption: " + slavePosition3 + " rows: " + rows.size());
assertTrue(slavePosition3.newerThan(slavePosition2));
for ( long i = 0 ; i < DATA_SIZE + NEW_DATA_SIZE + NEW_DATA_SIZE; i++ ) {
assertEquals(i + 1, rows.get((int) i).getData("id"));
}
}
@Test
public void testSchemaIdRestore() throws Exception {
MysqlIsolatedServer server = masterServer;
Position oldlogPosition = MaxwellTestSupport.capture(server.getConnection());
LOGGER.info("Initial pos: " + oldlogPosition);
MaxwellContext context = getContext(server.getPort(), false);
context.getPositionStore().set(oldlogPosition);
MysqlSavedSchema savedSchema = MysqlSavedSchema.restore(context, oldlogPosition);
if (savedSchema == null) {
Connection c = context.getMaxwellConnection();
Schema newSchema = new SchemaCapturer(c, context.getCaseSensitivity()).capture();
savedSchema = new MysqlSavedSchema(context, newSchema, context.getInitialPosition());
savedSchema.save(c);
}
Long oldSchemaId = savedSchema.getSchemaID();
LOGGER.info("old schema id: " + oldSchemaId);
server.execute("CREATE TABLE shard_1.new (id int(11))");
BufferedMaxwell maxwell = new BufferedMaxwell(getConfig(server.getPort(), false));
List<RowMap> rows = new ArrayList<>();
new Thread(maxwell).start();
drainReplication(maxwell, rows);
maxwell.terminate();
Position newPosition = MaxwellTestSupport.capture(server.getConnection());
LOGGER.info("New pos: " + newPosition);
MysqlSavedSchema newSavedSchema = MysqlSavedSchema.restore(context, newPosition);
LOGGER.info("New schema id: " + newSavedSchema.getSchemaID());
assertEquals(new Long(oldSchemaId + 1), newSavedSchema.getSchemaID());
assertTrue(newPosition.newerThan(savedSchema.getPosition()));
MysqlSavedSchema restored = MysqlSavedSchema.restore(context, oldlogPosition);
assertEquals(oldSchemaId, restored.getSchemaID());
}
}