/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core.snapshots;
import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.ListSnapshots;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.cloud.AbstractDistribZkTestBase;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Replica.State;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;
import org.apache.solr.handler.BackupRestoreUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
@Slow
public class TestSolrCloudSnapshots extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static long docsSeed; // see indexDocs()
private static final int NUM_SHARDS = 2;
private static final int NUM_REPLICAS = 2;
private static final int NUM_NODES = NUM_REPLICAS * NUM_SHARDS;
@BeforeClass
public static void setupClass() throws Exception {
useFactory("solr.StandardDirectoryFactory");
configureCluster(NUM_NODES)// nodes
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
.configure();
docsSeed = random().nextLong();
}
@AfterClass
public static void teardownClass() throws Exception {
System.clearProperty("test.build.data");
System.clearProperty("test.cache.data");
}
@Test
public void testSnapshots() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
String collectionName = "SolrCloudSnapshots";
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", NUM_SHARDS, NUM_REPLICAS);
create.process(solrClient);
int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed);
BackupRestoreUtils.verifyDocs(nDocs, solrClient, collectionName);
String commitName = TestUtil.randomSimpleString(random(), 1, 5);
// Verify if snapshot creation works with replica failures.
boolean replicaFailures = usually();
Optional<String> stoppedCoreName = Optional.empty();
if (replicaFailures) {
// Here the assumption is that Solr will spread the replicas uniformly across nodes.
// If this is not true for some reason, then we will need to add some logic to find a
// node with a single replica.
this.cluster.getRandomJetty(random()).stop();
// Sleep a bit for allowing ZK watch to fire.
Thread.sleep(5000);
// Figure out if at-least one replica is "down".
DocCollection collState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
for (Slice s : collState.getSlices()) {
for (Replica replica : s.getReplicas()) {
if (replica.getState() == State.DOWN) {
stoppedCoreName = Optional.of(replica.getCoreName());
}
}
}
}
int expectedCoresWithSnapshot = stoppedCoreName.isPresent() ? (NUM_SHARDS * NUM_REPLICAS) - 1 : (NUM_SHARDS * NUM_REPLICAS);
CollectionAdminRequest.CreateSnapshot createSnap = new CollectionAdminRequest.CreateSnapshot(collectionName, commitName);
createSnap.process(solrClient);
Collection<CollectionSnapshotMetaData> collectionSnaps = listCollectionSnapshots(solrClient, collectionName);
assertEquals(1, collectionSnaps.size());
CollectionSnapshotMetaData meta = collectionSnaps.iterator().next();
assertEquals(commitName, meta.getName());
assertEquals(CollectionSnapshotMetaData.SnapshotStatus.Successful, meta.getStatus());
assertEquals(expectedCoresWithSnapshot, meta.getReplicaSnapshots().size());
Map<String, CoreSnapshotMetaData> snapshotByCoreName = meta.getReplicaSnapshots().stream()
.collect(Collectors.toMap(CoreSnapshotMetaData::getCoreName, Function.identity()));
DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
assertEquals(2, collectionState.getActiveSlices().size());
for ( Slice shard : collectionState.getActiveSlices() ) {
assertEquals(2, shard.getReplicas().size());
for (Replica replica : shard.getReplicas()) {
if (stoppedCoreName.isPresent() && stoppedCoreName.get().equals(replica.getCoreName())) {
continue; // We know that the snapshot is not created for this replica.
}
String replicaBaseUrl = replica.getStr(BASE_URL_PROP);
String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
assertTrue(snapshotByCoreName.containsKey(coreName));
CoreSnapshotMetaData coreSnapshot = snapshotByCoreName.get(coreName);
try (SolrClient adminClient = getHttpSolrClient(replicaBaseUrl)) {
Collection<SnapshotMetaData> snapshots = listCoreSnapshots(adminClient, coreName);
Optional<SnapshotMetaData> metaData = snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst();
assertTrue("Snapshot not created for core " + coreName, metaData.isPresent());
assertEquals(coreSnapshot.getIndexDirPath(), metaData.get().getIndexDirPath());
assertEquals(coreSnapshot.getGenerationNumber(), metaData.get().getGenerationNumber());
}
}
}
// Delete all documents.
{
solrClient.deleteByQuery(collectionName, "*:*");
solrClient.commit(collectionName);
BackupRestoreUtils.verifyDocs(0, solrClient, collectionName);
}
String backupLocation = createTempDir().toFile().getAbsolutePath();
String backupName = "mytestbackup";
String restoreCollectionName = collectionName + "_restored";
//Create a backup using the earlier created snapshot.
{
CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
.setLocation(backupLocation).setCommitName(commitName);
if (random().nextBoolean()) {
assertEquals(0, backup.process(solrClient).getStatus());
} else {
assertEquals(RequestStatusState.COMPLETED, backup.processAndWait(solrClient, 30));//async
}
}
// Restore backup.
{
CollectionAdminRequest.Restore restore = CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)
.setLocation(backupLocation);
if (replicaFailures) {
// In this case one of the Solr servers would be down. Hence we need to increase
// max_shards_per_node property for restore command to succeed.
restore.setMaxShardsPerNode(2);
}
if (random().nextBoolean()) {
assertEquals(0, restore.process(solrClient).getStatus());
} else {
assertEquals(RequestStatusState.COMPLETED, restore.processAndWait(solrClient, 30));//async
}
AbstractDistribZkTestBase.waitForRecoveriesToFinish(
restoreCollectionName, cluster.getSolrClient().getZkStateReader(), log.isDebugEnabled(), true, 30);
BackupRestoreUtils.verifyDocs(nDocs, solrClient, restoreCollectionName);
}
// Verify if the snapshot deletion works correctly when one or more replicas containing the snapshot are
// deleted
boolean replicaDeletion = rarely();
if (replicaDeletion) {
CoreSnapshotMetaData replicaToDelete = null;
for (String shardId : meta.getShards()) {
List<CoreSnapshotMetaData> replicas = meta.getReplicaSnapshotsForShard(shardId);
if (replicas.size() > 1) {
int r_index = random().nextInt(replicas.size());
replicaToDelete = replicas.get(r_index);
}
}
if (replicaToDelete != null) {
collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
for (Slice s : collectionState.getSlices()) {
for (Replica r : s.getReplicas()) {
if (r.getCoreName().equals(replicaToDelete.getCoreName())) {
log.info("Deleting replica {}", r);
CollectionAdminRequest.DeleteReplica delReplica = CollectionAdminRequest.deleteReplica(collectionName,
replicaToDelete.getShardId(), r.getName());
delReplica.process(solrClient);
// The replica deletion will cleanup the snapshot meta-data.
snapshotByCoreName.remove(r.getCoreName());
break;
}
}
}
}
}
// Delete snapshot
CollectionAdminRequest.DeleteSnapshot deleteSnap = new CollectionAdminRequest.DeleteSnapshot(collectionName, commitName);
deleteSnap.process(solrClient);
// Wait for a while so that the clusterstate.json updates are propagated to the client side.
Thread.sleep(2000);
collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
for ( Slice shard : collectionState.getActiveSlices() ) {
for (Replica replica : shard.getReplicas()) {
if (stoppedCoreName.isPresent() && stoppedCoreName.get().equals(replica.getCoreName())) {
continue; // We know that the snapshot was not created for this replica.
}
String replicaBaseUrl = replica.getStr(BASE_URL_PROP);
String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
try (SolrClient adminClient = getHttpSolrClient(replicaBaseUrl)) {
Collection<SnapshotMetaData> snapshots = listCoreSnapshots(adminClient, coreName);
Optional<SnapshotMetaData> metaData = snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst();
assertFalse("Snapshot not deleted for core " + coreName, metaData.isPresent());
// Remove the entry for core if the snapshot is deleted successfully.
snapshotByCoreName.remove(coreName);
}
}
}
// Verify all core-level snapshots are deleted.
assertTrue("The cores remaining " + snapshotByCoreName, snapshotByCoreName.isEmpty());
assertTrue(listCollectionSnapshots(solrClient, collectionName).isEmpty());
// Verify if the collection deletion result in proper cleanup of snapshot metadata.
{
String commitName_2 = commitName + "_2";
CollectionAdminRequest.CreateSnapshot createSnap_2 = new CollectionAdminRequest.CreateSnapshot(collectionName, commitName_2);
assertEquals(0, createSnap_2.process(solrClient).getStatus());
Collection<CollectionSnapshotMetaData> collectionSnaps_2 = listCollectionSnapshots(solrClient, collectionName);
assertEquals(1, collectionSnaps.size());
assertEquals(commitName_2, collectionSnaps_2.iterator().next().getName());
// Delete collection
CollectionAdminRequest.Delete deleteCol = CollectionAdminRequest.deleteCollection(collectionName);
assertEquals(0, deleteCol.process(solrClient).getStatus());
assertTrue(SolrSnapshotManager.listSnapshots(solrClient.getZkStateReader().getZkClient(), collectionName).isEmpty());
}
}
private Collection<CollectionSnapshotMetaData> listCollectionSnapshots(SolrClient adminClient, String collectionName) throws Exception {
CollectionAdminRequest.ListSnapshots listSnapshots = new CollectionAdminRequest.ListSnapshots(collectionName);
CollectionAdminResponse resp = listSnapshots.process(adminClient);
assertTrue( resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO) instanceof NamedList );
NamedList apiResult = (NamedList) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
Collection<CollectionSnapshotMetaData> result = new ArrayList<>();
for (int i = 0; i < apiResult.size(); i++) {
result.add(new CollectionSnapshotMetaData((NamedList<Object>)apiResult.getVal(i)));
}
return result;
}
private Collection<SnapshotMetaData> listCoreSnapshots(SolrClient adminClient, String coreName) throws Exception {
ListSnapshots req = new ListSnapshots();
req.setCoreName(coreName);
NamedList resp = adminClient.request(req);
assertTrue( resp.get(SolrSnapshotManager.SNAPSHOTS_INFO) instanceof NamedList );
NamedList apiResult = (NamedList) resp.get(SolrSnapshotManager.SNAPSHOTS_INFO);
List<SnapshotMetaData> result = new ArrayList<>(apiResult.size());
for(int i = 0 ; i < apiResult.size(); i++) {
String commitName = apiResult.getName(i);
String indexDirPath = (String)((NamedList)apiResult.get(commitName)).get(SolrSnapshotManager.INDEX_DIR_PATH);
long genNumber = Long.parseLong((String)((NamedList)apiResult.get(commitName)).get(SolrSnapshotManager.GENERATION_NUM));
result.add(new SnapshotMetaData(commitName, indexDirPath, genNumber));
}
return result;
}
}