/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.util.TestUtil;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.segments.IndexSegments;
import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
import org.elasticsearch.action.admin.indices.segments.ShardSegments;
import org.elasticsearch.action.admin.indices.upgrade.get.IndexUpgradeStatus;
import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.engine.Segment;
import org.elasticsearch.index.shard.MergePolicyConfig;
import org.elasticsearch.indices.recovery.RecoverySettings;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.DirectoryStream;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import static junit.framework.TestCase.assertFalse;
import static junit.framework.TestCase.assertTrue;
import static org.elasticsearch.test.ESTestCase.randomInt;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.junit.Assert.assertEquals;
public class OldIndexUtils {
public static List<String> loadIndexesList(String prefix, Path bwcIndicesPath) throws IOException {
List<String> indexes = new ArrayList<>();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(bwcIndicesPath, prefix + "-*.zip")) {
for (Path path : stream) {
indexes.add(path.getFileName().toString());
}
}
Collections.sort(indexes);
return indexes;
}
public static Settings getSettings() {
return Settings.builder()
.put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) // disable merging so no segments will be upgraded
.put(RecoverySettings.INDICES_RECOVERY_CONCURRENT_SMALL_FILE_STREAMS, 30) // increase recovery speed for small files
.build();
}
public static void loadIndex(String indexName, String indexFile, Path unzipDir, Path bwcPath, ESLogger logger, Path... paths) throws
Exception {
Path unzipDataDir = unzipDir.resolve("data");
Path backwardsIndex = bwcPath.resolve(indexFile);
// decompress the index
try (InputStream stream = Files.newInputStream(backwardsIndex)) {
TestUtil.unzip(stream, unzipDir);
}
// check it is unique
assertTrue(Files.exists(unzipDataDir));
Path[] list = FileSystemUtils.files(unzipDataDir);
if (list.length != 1) {
throw new IllegalStateException("Backwards index must contain exactly one cluster");
}
// the bwc scripts packs the indices under this path
Path src = list[0].resolve("nodes/0/indices/" + indexName);
assertTrue("[" + indexFile + "] missing index dir: " + src.toString(), Files.exists(src));
copyIndex(logger, src, indexName, paths);
}
public static void assertNotUpgraded(Client client, String... index) throws Exception {
for (IndexUpgradeStatus status : getUpgradeStatus(client, index)) {
assertTrue("index " + status.getIndex() + " should not be zero sized", status.getTotalBytes() != 0);
// TODO: it would be better for this to be strictly greater, but sometimes an extra flush
// mysteriously happens after the second round of docs are indexed
assertTrue("index " + status.getIndex() + " should have recovered some segments from transaction log",
status.getTotalBytes() >= status.getToUpgradeBytes());
assertTrue("index " + status.getIndex() + " should need upgrading", status.getToUpgradeBytes() != 0);
}
}
@SuppressWarnings("unchecked")
public static Collection<IndexUpgradeStatus> getUpgradeStatus(Client client, String... indices) throws Exception {
UpgradeStatusResponse upgradeStatusResponse = client.admin().indices().prepareUpgradeStatus(indices).get();
assertNoFailures(upgradeStatusResponse);
return upgradeStatusResponse.getIndices().values();
}
// randomly distribute the files from src over dests paths
public static void copyIndex(final ESLogger logger, final Path src, final String indexName, final Path... dests) throws IOException {
for (Path dest : dests) {
Path indexDir = dest.resolve(indexName);
assertFalse(Files.exists(indexDir));
Files.createDirectories(indexDir);
}
Files.walkFileTree(src, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
Path relativeDir = src.relativize(dir);
for (Path dest : dests) {
Path destDir = dest.resolve(indexName).resolve(relativeDir);
Files.createDirectories(destDir);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (file.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME)) {
// skip lock file, we don't need it
logger.trace("Skipping lock file: " + file.toString());
return FileVisitResult.CONTINUE;
}
Path relativeFile = src.relativize(file);
Path destFile = dests[randomInt(dests.length - 1)].resolve(indexName).resolve(relativeFile);
logger.trace("--> Moving " + relativeFile.toString() + " to " + destFile.toString());
Files.move(file, destFile);
assertFalse(Files.exists(file));
assertTrue(Files.exists(destFile));
return FileVisitResult.CONTINUE;
}
});
}
public static void assertUpgraded(Client client, String... index) throws Exception {
for (IndexUpgradeStatus status : getUpgradeStatus(client, index)) {
assertTrue("index " + status.getIndex() + " should not be zero sized", status.getTotalBytes() != 0);
assertEquals("index " + status.getIndex() + " should be upgraded",
0, status.getToUpgradeBytes());
}
// double check using the segments api that all segments are actually upgraded
IndicesSegmentResponse segsRsp;
if (index == null) {
segsRsp = client.admin().indices().prepareSegments().execute().actionGet();
} else {
segsRsp = client.admin().indices().prepareSegments(index).execute().actionGet();
}
for (IndexSegments indexSegments : segsRsp.getIndices().values()) {
for (IndexShardSegments shard : indexSegments) {
for (ShardSegments segs : shard.getShards()) {
for (Segment seg : segs.getSegments()) {
assertEquals("Index " + indexSegments.getIndex() + " has unupgraded segment " + seg.toString(),
Version.CURRENT.luceneVersion.major, seg.version.major);
assertEquals("Index " + indexSegments.getIndex() + " has unupgraded segment " + seg.toString(),
Version.CURRENT.luceneVersion.minor, seg.version.minor);
}
}
}
}
}
public static void assertUpgradeWorks(Client client, String indexName, Version version) throws Exception {
if (OldIndexUtils.isLatestLuceneVersion(version) == false) {
OldIndexUtils.assertNotUpgraded(client, indexName);
}
assertNoFailures(client.admin().indices().prepareUpgrade(indexName).get());
assertUpgraded(client, indexName);
}
public static Version extractVersion(String index) {
return Version.fromString(index.substring(index.indexOf('-') + 1, index.lastIndexOf('.')));
}
public static boolean isLatestLuceneVersion(Version version) {
return version.luceneVersion.major == Version.CURRENT.luceneVersion.major &&
version.luceneVersion.minor == Version.CURRENT.luceneVersion.minor;
}
}