/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.util.FileUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
public class TestRestoreCore extends SolrJettyTestBase {
JettySolrRunner masterJetty;
TestReplicationHandler.SolrInstance master = null;
SolrClient masterClient;
private static final String CONF_DIR = "solr" + File.separator + DEFAULT_TEST_CORENAME + File.separator + "conf"
+ File.separator;
private static String context = "/solr";
private static long docsSeed; // see indexDocs()
private static JettySolrRunner createJetty(TestReplicationHandler.SolrInstance instance) throws Exception {
FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(instance.getHomeDir(), "solr.xml"));
Properties nodeProperties = new Properties();
nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
JettyConfig jettyConfig = JettyConfig.builder().setContext("/solr").setPort(0).build();
JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, jettyConfig);
jetty.start();
return jetty;
}
private static SolrClient createNewSolrClient(int port) {
try {
// setup the client...
final String baseUrl = buildUrl(port, context);
HttpSolrClient client = getHttpSolrClient(baseUrl);
client.setConnectionTimeout(15000);
client.setSoTimeout(60000);
return client;
}
catch (Exception ex) {
throw new RuntimeException(ex);
}
}
@Before
public void setUp() throws Exception {
super.setUp();
String configFile = "solrconfig-master.xml";
master = new TestReplicationHandler.SolrInstance(createTempDir("solr-instance").toFile(), "master", null);
master.setUp();
master.copyConfigFile(CONF_DIR + configFile, "solrconfig.xml");
masterJetty = createJetty(master);
masterClient = createNewSolrClient(masterJetty.getLocalPort());
docsSeed = random().nextLong();
}
@Override
@After
public void tearDown() throws Exception {
super.tearDown();
masterClient.close();
masterClient = null;
masterJetty.stop();
masterJetty = null;
master = null;
}
@Test
public void testSimpleRestore() throws Exception {
int nDocs = usually() ? BackupRestoreUtils.indexDocs(masterClient, "collection1", docsSeed) : 0;
String snapshotName;
String location;
String params = "";
String baseUrl = masterJetty.getBaseUrl().toString();
//Use the default backup location or an externally provided location.
if (random().nextBoolean()) {
location = createTempDir().toFile().getAbsolutePath();
params += "&location=" + URLEncoder.encode(location, "UTF-8");
}
//named snapshot vs default snapshot name
if (random().nextBoolean()) {
snapshotName = TestUtil.randomSimpleString(random(), 1, 5);
params += "&name=" + snapshotName;
}
TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_BACKUP, params);
CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, DEFAULT_TEST_CORENAME, null);
while (!checkBackupStatus.success) {
checkBackupStatus.fetchStatus();
Thread.sleep(1000);
}
int numRestoreTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1;
for (int attempts=0; attempts<numRestoreTests; attempts++) {
//Modify existing index before we call restore.
if (nDocs > 0) {
//Delete a few docs
int numDeletes = TestUtil.nextInt(random(), 1, nDocs);
for(int i=0; i<numDeletes; i++) {
masterClient.deleteByQuery(DEFAULT_TEST_CORENAME, "id:" + i);
}
masterClient.commit(DEFAULT_TEST_CORENAME);
//Add a few more
int moreAdds = TestUtil.nextInt(random(), 1, 100);
for (int i=0; i<moreAdds; i++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", i + nDocs);
doc.addField("name", "name = " + (i + nDocs));
masterClient.add(DEFAULT_TEST_CORENAME, doc);
}
//Purposely not calling commit once in a while. There can be some docs which are not committed
if (usually()) {
masterClient.commit(DEFAULT_TEST_CORENAME);
}
}
TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_RESTORE, params);
while (!fetchRestoreStatus(baseUrl, DEFAULT_TEST_CORENAME)) {
Thread.sleep(1000);
}
//See if restore was successful by checking if all the docs are present again
BackupRestoreUtils.verifyDocs(nDocs, masterClient, DEFAULT_TEST_CORENAME);
}
}
@Test
public void testFailedRestore() throws Exception {
int nDocs = BackupRestoreUtils.indexDocs(masterClient, "collection1", docsSeed);
String location = createTempDir().toFile().getAbsolutePath();
String snapshotName = TestUtil.randomSimpleString(random(), 1, 5);
String params = "&name=" + snapshotName + "&location=" + URLEncoder.encode(location, "UTF-8");
String baseUrl = masterJetty.getBaseUrl().toString();
TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_BACKUP, params);
CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, DEFAULT_TEST_CORENAME, null);
while (!checkBackupStatus.success) {
checkBackupStatus.fetchStatus();
Thread.sleep(1000);
}
//Remove the segments_n file so that the backup index is corrupted.
//Restore should fail and it should automatically rollback to the original index.
Path restoreIndexPath = Paths.get(location).resolve("snapshot." + snapshotName);
try (DirectoryStream<Path> stream = Files.newDirectoryStream(restoreIndexPath, IndexFileNames.SEGMENTS + "*")) {
Path segmentFileName = stream.iterator().next();
Files.delete(segmentFileName);
}
TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_RESTORE, params);
try {
while (!fetchRestoreStatus(baseUrl, DEFAULT_TEST_CORENAME)) {
Thread.sleep(1000);
}
fail("Should have thrown an error because restore could not have been successful");
} catch (AssertionError e) {
//supposed to happen
}
BackupRestoreUtils.verifyDocs(nDocs, masterClient, DEFAULT_TEST_CORENAME);
//make sure we can write to the index again
nDocs = BackupRestoreUtils.indexDocs(masterClient, "collection1", docsSeed);
BackupRestoreUtils.verifyDocs(nDocs, masterClient, DEFAULT_TEST_CORENAME);
}
public static boolean fetchRestoreStatus (String baseUrl, String coreName) throws IOException {
String masterUrl = baseUrl + "/" + coreName +
ReplicationHandler.PATH + "?command=" + ReplicationHandler.CMD_RESTORE_STATUS;
final Pattern pException = Pattern.compile("<str name=\"exception\">(.*?)</str>");
InputStream stream = null;
try {
URL url = new URL(masterUrl);
stream = url.openStream();
String response = IOUtils.toString(stream, "UTF-8");
Matcher matcher = pException.matcher(response);
if(matcher.find()) {
fail("Failed to complete restore action with exception " + matcher.group(1));
}
if(response.contains("<str name=\"status\">success</str>")) {
return true;
} else if (response.contains("<str name=\"status\">failed</str>")){
fail("Restore Failed");
}
stream.close();
} finally {
IOUtils.closeQuietly(stream);
}
return false;
}
}