/*
* Copyright (c) 2017 Strapdata (http://www.strapdata.com)
* Contains some code from Elasticsearch (http://www.elastic.co)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.elassandra;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.Date;
import java.util.Locale;
import java.util.UUID;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.service.StorageService;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.junit.Test;
/**
* Elassandra snapshot tests.
* @author vroyer
*
*/
//mvn test -Pdev -pl om.strapdata.elasticsearch:elasticsearch -Dtests.seed=622A2B0618CE4676 -Dtests.class=org.elassandra.SnapshotTests -Des.logger.level=ERROR -Dtests.assertion.disabled=false -Dtests.security.manager=false -Dtests.heap.size=1024m -Dtests.locale=ro-RO -Dtests.timezone=America/Toronto
public class SnapshotTests extends ESSingleNodeTestCase {
// SSTable snapshotDir = data/data/<keyspace>/<table>/snapshots/<snapshot_name>/
public void restoreSSTable(String dataLocation, String keyspaceName, String cfName, UUID cfId, String snapshotName) throws IOException {
String id = cfId.toString().replaceAll("\\-", "");
Path sourceDir = PathUtils.get(dataLocation+"/"+keyspaceName+"/"+cfName+"-"+id+"/snapshots/"+snapshotName);
Path targetDir = PathUtils.get(dataLocation+"/"+keyspaceName+"/"+cfName+"-"+id+"/");
try (DirectoryStream<Path> stream = Files.newDirectoryStream(targetDir, "{*.db}")) {
for (Path dbFile: stream)
Files.delete(dbFile);
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(sourceDir)) {
for (Path f: stream)
Files.copy(f, PathUtils.get(targetDir.toString(), f.getFileName().toString()) , StandardCopyOption.COPY_ATTRIBUTES);
}
}
// lucene snapshotDir = data/<cluster_name>/nodes/0/snapshots/<index_name>/<snapshot_name>
public void restoreLucenceFiles(String dataLocation, String indexName, String snapshot) throws IOException {
Path sourceDir = PathUtils.get(dataLocation+"/"+DatabaseDescriptor.getClusterName()+"/nodes/0/snapshots/"+indexName+"/"+snapshot+"/");
Path targetDir = PathUtils.get(dataLocation+"/"+DatabaseDescriptor.getClusterName()+"/nodes/0/indices/"+indexName+"/0/index/");
try (DirectoryStream<Path> stream = Files.newDirectoryStream(targetDir, "{_*.*,segments*}")) {
for (Path segmentFile: stream)
Files.delete(segmentFile);
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(sourceDir, "{_*.*,segments*}")) {
for (Path f: stream)
Files.copy(f, PathUtils.get(targetDir.toString(), f.getFileName().toString()) , StandardCopyOption.COPY_ATTRIBUTES);
}
}
@Test
public void basicSnapshotTest() throws Exception {
process(ConsistencyLevel.ONE,String.format(Locale.ROOT, "CREATE KEYSPACE ks WITH replication = {'class': 'NetworkTopologyStrategy', '%s': '1'}",DatabaseDescriptor.getLocalDataCenter()));
process(ConsistencyLevel.ONE,"CREATE TABLE ks.t1 ( name text, age int, primary key (name))");
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("t1").field("discover", ".*").endObject().endObject();
createIndex("ks", Settings.builder().put("index.snapshot_with_sstable",true).build(),"t1", mapping);
ensureGreen("ks");
for(long i=0; i < 1000; i++)
process(ConsistencyLevel.ONE,String.format(Locale.ROOT, "INSERT INTO ks.t1 (name, age) VALUES ('name%d', %d)",i,i));
assertThat(client().prepareSearch().setIndices("ks").setTypes("t1").setQuery(QueryBuilders.queryStringQuery("*:*")).get().getHits().getTotalHits(), equalTo(1000L));
// take snaphot
StorageService.instance.takeSnapshot("snap1", "ks");
// drop all
process(ConsistencyLevel.ONE,"TRUNCATE ks.t1");
assertThat(client().prepareSearch().setIndices("ks").setTypes("t1").setQuery(QueryBuilders.queryStringQuery("*:*")).get().getHits().getTotalHits(), equalTo(0L));
// close index and restore SSTable+Lucene files
assertAcked(client().admin().indices().prepareClose("ks").get());
String dataLocation = DatabaseDescriptor.getAllDataFileLocations()[0];
restoreSSTable(dataLocation, "ks", "t1", Schema.instance.getCFMetaData("ks", "t1").cfId, "snap1");
restoreLucenceFiles(dataLocation, "ks", "snap1");
// refresh SSTables and repopen index
StorageService.instance.loadNewSSTables("ks", "t1");
assertAcked(client().admin().indices().prepareOpen("ks").get());
ensureGreen("ks");
assertThat(client().prepareSearch().setIndices("ks").setTypes("t1").setQuery(QueryBuilders.queryStringQuery("*:*")).get().getHits().getTotalHits(), equalTo(1000L));
}
@Test
//mvn test -Pdev -pl com.strapdata.elasticsearch:elasticsearch -Dtests.seed=622A2B0618CE4676 -Dtests.class=org.elassandra.SnapshotTests -Dtests.method="onDropSnapshotTest" -Des.logger.level=ERROR -Dtests.assertion.disabled=false -Dtests.security.manager=false -Dtests.heap.size=1024m -Dtests.locale=ro-RO -Dtests.timezone=America/Toronto
public void onDropSnapshotTest() throws Exception {
process(ConsistencyLevel.ONE,String.format(Locale.ROOT, "CREATE KEYSPACE ks WITH replication = {'class': 'NetworkTopologyStrategy', '%s': '1'}",DatabaseDescriptor.getLocalDataCenter()));
process(ConsistencyLevel.ONE,"CREATE TABLE ks.t1 ( name text, age int, primary key (name))");
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("t1").field("discover", ".*").endObject().endObject();
createIndex("ks", Settings.builder().put("index.snapshot_with_sstable",true).build(),"t1", mapping);
ensureGreen("ks");
for(long i=0; i < 1000; i++)
process(ConsistencyLevel.ONE,String.format(Locale.ROOT, "INSERT INTO ks.t1 (name, age) VALUES ('name%d', %d)",i,i));
assertThat(client().prepareSearch().setIndices("ks").setTypes("t1").setQuery(QueryBuilders.queryStringQuery("*:*")).get().getHits().getTotalHits(), equalTo(1000L));
UUID cfId = Schema.instance.getCFMetaData("ks","t1").cfId;
String id = cfId.toString().replaceAll("\\-", "");
if (!DatabaseDescriptor.isAutoSnapshot())
StorageService.instance.takeTableSnapshot("ks", "t1", Long.toString(new Date().getTime()));
// drop index + keyspace (C* snapshot before drop => flush before snapshot => ES flush before delete)
assertAcked(client().admin().indices().prepareDelete("ks").get());
// recreate schema and mapping
process(ConsistencyLevel.ONE,String.format(Locale.ROOT, "CREATE KEYSPACE ks WITH replication = {'class': 'NetworkTopologyStrategy', '%s': '1'}",DatabaseDescriptor.getLocalDataCenter()));
process(ConsistencyLevel.ONE,"CREATE TABLE ks.t1 ( name text, age int, primary key (name))");
createIndex("ks", Settings.builder().put("index.snapshot_with_sstable",true).build(),"t1", mapping);
ensureGreen("ks");
assertThat(client().prepareSearch().setIndices("ks").setTypes("t1").setQuery(QueryBuilders.queryStringQuery("*:*")).get().getHits().getTotalHits(), equalTo(0L));
// close index and restore SSTable+Lucene files
assertAcked(client().admin().indices().prepareClose("ks").get());
String dataLocation = DatabaseDescriptor.getAllDataFileLocations()[0];
DirectoryStream<Path> stream = Files.newDirectoryStream(PathUtils.get(dataLocation+"/ks/t1-"+id+"/snapshots/"));
Path snapshot = stream.iterator().next();
String snap = snapshot.getFileName().toString();
System.out.println("snapshot name="+snap);
stream.close();
restoreSSTable(dataLocation, "ks", "t1", cfId, snap);
restoreLucenceFiles(dataLocation, "ks", snap);
// refresh SSTables and repopen index
StorageService.instance.loadNewSSTables("ks", "t1");
assertAcked(client().admin().indices().prepareOpen("ks").get());
ensureGreen("ks");
assertThat(client().prepareSearch().setIndices("ks").setTypes("t1").setQuery(QueryBuilders.queryStringQuery("*:*")).get().getHits().getTotalHits(), equalTo(1000L));
}
}