/*
* The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
* (the "License"). You may not use this work except in compliance with the License, which is
* available at www.apache.org/licenses/LICENSE-2.0
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied, as more fully set forth in the License.
*
* See the NOTICE file distributed with this work for information regarding copyright ownership.
*/
package alluxio.master.journal;
import alluxio.AlluxioURI;
import alluxio.Configuration;
import alluxio.PropertyKey;
import alluxio.RuntimeConstants;
import alluxio.ServiceUtils;
import alluxio.master.MasterFactory;
import alluxio.underfs.UnderFileSystem;
import alluxio.underfs.options.MkdirsOptions;
import alluxio.util.URIUtils;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.concurrent.NotThreadSafe;
/**
* Tool to upgrade journal from v0 to v1. Format the v1 journal before running this tool.
*
* It is strongly recommended to backup the v0 journal before running this tool to avoid losing
* any data in case of failures.
*
* <pre>
* java -cp \
* assembly/server/target/alluxio-assembly-server-<ALLUXIO-VERSION>-jar-with-dependencies.jar \
* alluxio.master.journal.JournalUpgrader -journalDirectoryV0 YourJournalDirectoryV0
* </pre>
*/
@NotThreadSafe
public final class JournalUpgrader {
private static final Logger LOG = LoggerFactory.getLogger(JournalUpgrader.class);
private static final int EXIT_FAILED = -1;
private static final int EXIT_SUCCEEDED = 0;
private static final Options OPTIONS =
new Options().addOption("help", false, "Show help for this test.")
.addOption("journalDirectoryV0", true,
"Where the v0 journal is persisted. It is assumed to be the same as the v1 journal "
+ "directory if not set.");
private static boolean sHelp;
private static String sJournalDirectoryV0;
/**
* A class that provides a way to upgrade the journal from v0 to v1.
*/
private static final class Upgrader {
private final String mMaster;
private final alluxio.master.journalv0.MutableJournal mJournalV0;
private final Journal mJournalV1;
private final UnderFileSystem mUfs;
private final URI mCheckpointV0;
private final URI mCompletedLogsV0;
private final URI mCheckpointsV1;
private final URI mLogsV1;
private Upgrader(String master) {
mMaster = master;
mJournalV0 = (new alluxio.master.journalv0.MutableJournal.Factory(
getJournalLocation(sJournalDirectoryV0))).create(master);
mJournalV1 = (new Journal.Factory(
getJournalLocation(Configuration.get(PropertyKey.MASTER_JOURNAL_FOLDER)))).create(master);
mUfs = UnderFileSystem.Factory.create(sJournalDirectoryV0);
mCheckpointV0 = URIUtils.appendPathOrDie(mJournalV0.getLocation(), "checkpoint.data");
mCompletedLogsV0 = URIUtils.appendPathOrDie(mJournalV0.getLocation(), "completed");
mCheckpointsV1 = URIUtils.appendPathOrDie(mJournalV1.getLocation(), "checkpoints");
mLogsV1 = URIUtils.appendPathOrDie(mJournalV1.getLocation(), "logs");
}
/**
* Upgrades journal from v0 to v1.
*/
void upgrade() throws IOException {
if (!mUfs.exists(mCheckpointV0.toString())) {
LOG.info("No checkpoint is found for {}. No upgrade is required.", mMaster);
return;
}
prepare();
LOG.info("Starting to upgrade {} journal.", mMaster);
boolean checkpointUpgraded = false;
int logNumber = 1;
URI completedLog;
while (mUfs.exists((completedLog = getCompletedLogV0(logNumber)).toString())) {
long start = -1;
long end = -1;
logNumber++;
try (JournalFileParser parser = JournalFileParser.Factory.create(completedLog)) {
alluxio.proto.journal.Journal.JournalEntry entry;
while ((entry = parser.next()) != null) {
if (start == -1) {
start = entry.getSequenceNumber();
}
end = entry.getSequenceNumber();
}
}
if (!checkpointUpgraded) {
renameCheckpoint(start);
checkpointUpgraded = true;
}
URI dst = URIUtils.appendPathOrDie(mLogsV1, String.format("0x%x-0x%x", start, end + 1));
if (!mUfs.renameFile(completedLog.toString(), dst.toString()) && !mUfs
.exists(dst.toString())) {
throw new IOException(
String.format("Failed to rename %s to %s.", completedLog.toString(), dst.toString()));
}
}
if (!checkpointUpgraded) {
renameCheckpoint(1);
}
LOG.info("Finished upgrading {} journal.", mMaster);
}
/**
* Prepares journal writer to upgrade journals from v0 to v1.
*/
private void prepare() throws IOException {
alluxio.master.journalv0.JournalWriter journalWriterV0 = mJournalV0.getWriter();
journalWriterV0.recover();
journalWriterV0.completeLogs();
journalWriterV0.close();
if (!mJournalV1.isFormatted()) {
LOG.info("Starting to format journal {}.", mJournalV1.getLocation());
mJournalV1.format();
LOG.info("Finished formatting journal {}.", mJournalV1.getLocation());
}
if (!mUfs.exists(mCheckpointsV1.toString())) {
mUfs.mkdirs(mCheckpointsV1.toString(), MkdirsOptions.defaults().setCreateParent(true));
}
if (!mUfs.exists(mLogsV1.toString())) {
mUfs.mkdirs(mLogsV1.toString(), MkdirsOptions.defaults().setCreateParent(true));
}
}
/**
* Renames checkpoint.
*
* @param sequenceNumber the sequence number
*/
private void renameCheckpoint(long sequenceNumber) throws IOException {
URI dst = URIUtils.appendPathOrDie(mCheckpointsV1, String.format("0x0-0x%x", sequenceNumber));
if (!mUfs.renameFile(mCheckpointV0.toString(), dst.toString()) && !mUfs
.exists(dst.toString())) {
throw new IOException(
String.format("Failed to rename %s to %s.", mCheckpointV0.toString(), dst.toString()));
}
}
/**
* @param logNumber the log number to get the path for
* @return the location of the completed log for a particular log number
*/
private URI getCompletedLogV0(long logNumber) {
return URIUtils
.appendPathOrDie(mCompletedLogsV0, String.format("%s.%020d", "log", logNumber));
}
/**
* @return the journal location
*/
private URI getJournalLocation(String journalDirectory) {
if (!journalDirectory.endsWith(AlluxioURI.SEPARATOR)) {
journalDirectory += AlluxioURI.SEPARATOR;
}
try {
return new URI(journalDirectory);
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
}
private JournalUpgrader() {} // prevent instantiation
/**
* Reads a journal via
* {@code java -cp \
* assembly/server/target/alluxio-assembly-server-<ALLUXIO-VERSION>-jar-with-dependencies.jar \
* alluxio.master.journal.JournalUpgrader -master BlockMaster}.
*
* @param args arguments passed to the tool
*/
public static void main(String[] args) {
if (!parseInputArgs(args)) {
usage();
System.exit(EXIT_FAILED);
}
if (sHelp) {
usage();
System.exit(EXIT_SUCCEEDED);
}
List<String> masters = new ArrayList<>();
for (MasterFactory factory : ServiceUtils.getMasterServiceLoader()) {
masters.add(factory.getName());
}
for (String master : masters) {
Upgrader upgrader = new Upgrader(master);
try {
upgrader.upgrade();
} catch (IOException e) {
LOG.error("Failed to upgrade the journal for {}.", master, e);
System.exit(EXIT_FAILED);
}
}
}
/**
* Parses the input args with a command line format, using
* {@link org.apache.commons.cli.CommandLineParser}.
*
* @param args the input args
* @return true if parsing succeeded
*/
private static boolean parseInputArgs(String[] args) {
CommandLineParser parser = new DefaultParser();
CommandLine cmd;
try {
cmd = parser.parse(OPTIONS, args);
} catch (ParseException e) {
System.out.println("Failed to parse input args: " + e);
return false;
}
sHelp = cmd.hasOption("help");
sJournalDirectoryV0 = cmd.getOptionValue("journalDirectoryV0",
Configuration.get(PropertyKey.MASTER_JOURNAL_FOLDER));
return true;
}
/**
* Prints the usage.
*/
private static void usage() {
new HelpFormatter().printHelp("java -cp alluxio-" + RuntimeConstants.VERSION
+ "-jar-with-dependencies.jar alluxio.master.journal.JournalUpgrader",
"Upgrades journal from v0 to v1", OPTIONS, "", true);
}
}