/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.blur.shell;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.TreeMap;
import org.apache.blur.thirdparty.thrift_0_9_0.TException;
import org.apache.blur.thrift.generated.Blur.Iface;
import org.apache.blur.thrift.generated.BlurException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class DiscoverFileBufferSizeUtil extends Command {
@Override
public void doit(PrintWriter out, Iface client, String[] args) throws CommandException, TException, BlurException {
CommandLine cmd = parse(args, out);
if (cmd == null) {
throw new CommandException(name() + " missing required arguments");
}
Option[] options = cmd.getOptions();
for (Option option : options) {
out.println(option);
String opt = option.getOpt();
String value = option.getValue();
System.out.println(opt + " " + value);
}
Path path = new Path(cmd.getOptionValue("p"));
long size = Long.parseLong(cmd.getOptionValue("s", "1000000000"));// 1GB
int sampleSize = Integer.parseInt(cmd.getOptionValue("S", "10"));// 10
int warmupSize = Integer.parseInt(cmd.getOptionValue("W", "3"));// 3
int min = Integer.parseInt(cmd.getOptionValue("n", "12"));// 12
int max = Integer.parseInt(cmd.getOptionValue("x", "19"));// 19
int readSamplesPerPass = Integer.parseInt(cmd.getOptionValue("r", "1000"));// 1000
Configuration configuration = new Configuration();
FileSystem fileSystem;
try {
fileSystem = path.getFileSystem(configuration);
} catch (IOException e) {
if (Main.debug) {
e.printStackTrace();
}
throw new CommandException(e.getMessage());
}
Random random = new Random();
Map<Integer, Long> writingResults;
try {
writingResults = runWritingTest(out, path, fileSystem, random, sampleSize, warmupSize, size, min, max);
} catch (IOException e) {
if (Main.debug) {
e.printStackTrace();
}
throw new CommandException(e.getMessage());
}
Map<Integer, Long> readingResults;
try {
readingResults = runReadingTest(out, path, fileSystem, random, sampleSize, warmupSize, size, min, max,
readSamplesPerPass);
} catch (IOException e) {
if (Main.debug) {
e.printStackTrace();
}
throw new CommandException(e.getMessage());
}
out.println("Printing Results for Writing");
printResults(out, writingResults);
out.println("Printing Results for Reading");
printResults(out, readingResults);
}
@SuppressWarnings("static-access")
public CommandLine parse(String[] otherArgs, Writer out) {
Options options = new Options();
options.addOption(OptionBuilder.withDescription("The path to write temp files.").hasArg().isRequired(true)
.create("p"));
options.addOption(OptionBuilder.withDescription("The size of the temp files (1 GB by default).").hasArg()
.create("s"));
options.addOption(OptionBuilder.withDescription("Number of cycles of the test. (10 by default)").hasArg()
.create("S"));
options.addOption(OptionBuilder.withDescription("Number of warmup cycles. (3 by default)").hasArg().create("W"));
options.addOption(OptionBuilder.withDescription("Min buffer size power of 2 (12 by default 4KB)").hasArg()
.create("n"));
options.addOption(OptionBuilder.withDescription("Max buffer size power of 2 (19 by default 512KB)").hasArg()
.create("x"));
options.addOption(OptionBuilder
.withDescription("Number of random read samples during read test. (1000 by default)").hasArg().create("r"));
options.addOption(OptionBuilder.withDescription("Displays help for this command.").create("h"));
CommandLineParser parser = new PosixParser();
CommandLine cmd = null;
try {
cmd = parser.parse(options, otherArgs);
if (cmd.hasOption("h")) {
HelpFormatter formatter = new HelpFormatter();
PrintWriter pw = new PrintWriter(out, true);
formatter.printHelp(pw, HelpFormatter.DEFAULT_WIDTH, name(), null, options, HelpFormatter.DEFAULT_LEFT_PAD,
HelpFormatter.DEFAULT_DESC_PAD, null, false);
return null;
}
} catch (ParseException e) {
HelpFormatter formatter = new HelpFormatter();
PrintWriter pw = new PrintWriter(out, true);
formatter.printHelp(pw, HelpFormatter.DEFAULT_WIDTH, name(), null, options, HelpFormatter.DEFAULT_LEFT_PAD,
HelpFormatter.DEFAULT_DESC_PAD, null, false);
return null;
}
return cmd;
}
@Override
public String description() {
return "Tune the buffer sizes for your filesystem. Run -h for full argument list.";
}
@Override
public String usage() {
return "-p <tmp path>";
}
@Override
public String name() {
return "fstune";
}
private static void printResults(PrintWriter out, Map<Integer, Long> results) {
int bestBuffer = getTheBest(results);
for (Entry<Integer, Long> e : results.entrySet()) {
int bufSize = e.getKey();
Long time = e.getValue();
if (bufSize == bestBuffer) {
out.print("*");
} else {
out.print(" ");
}
out.println(" Buffer size [" + bufSize + "] took [" + (time) / 1000000.0 + " ms]");
out.flush();
}
}
private static Map<Integer, Long> runReadingTest(PrintWriter out, Path path, FileSystem fileSystem, Random random,
int sampleSize, int warmupSize, long size, int min, int max, int readSamples) throws IOException {
out.println("Warming up JVM for Reading.");
out.flush();
for (int i = 0; i < warmupSize; i++) {
testReadBuffer(out, "Warmup Read Pass [" + i + "]", random, fileSystem, path, size, min, max, readSamples);
}
Map<Integer, Long> readingResults = new TreeMap<Integer, Long>();
for (int s = 0; s < sampleSize; s++) {
add(readingResults,
testReadBuffer(out, "Read Pass [" + s + "]", random, fileSystem, path, size, min, max, readSamples));
}
return readingResults;
}
private static Map<Integer, Long> runWritingTest(PrintWriter out, Path path, FileSystem fileSystem, Random random,
int sampleSize, int warmupSize, long size, int min, int max) throws IOException {
out.println("Warming up JVM for Writing.");
out.flush();
for (int i = 0; i < warmupSize; i++) {
testWriteBuffer(out, "Warmup Write Pass [" + i + "]", random, fileSystem, path, size, min, max);
}
Map<Integer, Long> writingResults = new TreeMap<Integer, Long>();
for (int s = 0; s < sampleSize; s++) {
add(writingResults, testWriteBuffer(out, "Write Pass [" + s + "]", random, fileSystem, path, size, min, max));
}
return writingResults;
}
private static void add(Map<Integer, Long> results, Map<Integer, Long> newResults) {
for (Entry<Integer, Long> e : newResults.entrySet()) {
Long value = results.get(e.getKey());
if (value == null) {
results.put(e.getKey(), e.getValue());
} else {
results.put(e.getKey(), e.getValue() + value);
}
}
}
private static Integer getTheBest(Map<Integer, Long> results) {
long lowestTime = Long.MAX_VALUE;
Integer bestKey = null;
for (Entry<Integer, Long> e : results.entrySet()) {
Long value = e.getValue();
if (value < lowestTime) {
lowestTime = value;
bestKey = e.getKey();
}
}
return bestKey;
}
private static Map<Integer, Long> testWriteBuffer(PrintWriter out, String prefix, Random random,
FileSystem fileSystem, Path path, long size, int min, int max) throws IOException {
Map<Integer, Long> results = new TreeMap<Integer, Long>();
for (int i = min; i <= max; i++) {
int bufSize = (int) Math.pow(2, i);
out.println(prefix + " Creating [" + size + "] length file for write testing using buffer size of [" + bufSize
+ "]");
out.flush();
FSDataOutputStream outputStream = fileSystem.create(new Path(path, "test.data"), true, bufSize);
long time = writeFile(out, random, bufSize, outputStream, size);
outputStream.close();
results.put(bufSize, time);
out.println(prefix + " Buffer size [" + bufSize + "] took [" + time / 1000000.0 + " ms]");
out.flush();
}
return results;
}
private static Map<Integer, Long> testReadBuffer(PrintWriter out, String prefix, Random random,
FileSystem fileSystem, Path path, long length, int min, int max, int readSamples) throws IOException {
FSDataOutputStream outputStream = fileSystem.create(new Path(path, "test.data"), true, 8192);
writeFile(out, random, 8192, outputStream, length);
outputStream.close();
Map<Integer, Long> results = new TreeMap<Integer, Long>();
for (int i = min; i <= max; i++) {
int bufSize = (int) Math.pow(2, i);
out.println(prefix + " Reading [" + length + "] length file for reading test using buffer size of [" + bufSize
+ "]");
out.flush();
FSDataInputStream inputStream = fileSystem.open(new Path(path, "test.data"), bufSize);
long time = readFile(out, random, bufSize, inputStream, length, readSamples);
inputStream.close();
results.put(bufSize, time);
out.println(prefix + " Buffer size [" + bufSize + "] took [" + time / 1000000.0 + " ms]");
out.flush();
}
return results;
}
private static long readFile(PrintWriter out, Random random, int bufSize, FSDataInputStream inputStream, long length,
int readSamples) throws IOException {
byte[] buf = new byte[bufSize];
long start = System.nanoTime();
long time = 0;
for (int i = 0; i < readSamples; i++) {
long now = System.nanoTime();
if (start + 5000000000l < now) {
double complete = (((double) i / (double) readSamples) * 100.0);
out.println(complete + "% Complete");
out.flush();
start = System.nanoTime();
}
random.nextBytes(buf);
long position = getPosition(bufSize, random, length);
long s = System.nanoTime();
int offset = 0;
int len = bufSize;
while (len > 0) {
int amount = inputStream.read(position, buf, offset, len);
len -= amount;
offset += amount;
position += amount;
}
long e = System.nanoTime();
time += (e - s);
length -= len;
}
return time;
}
private static long getPosition(int bufSize, Random random, long length) {
return Math.abs(random.nextLong()) % (length - bufSize);
}
private static long writeFile(PrintWriter out, Random random, int bufSize, FSDataOutputStream outputStream,
long length) throws IOException {
byte[] buf = new byte[bufSize];
final long origLength = length;
long start = System.nanoTime();
long time = 0;
while (length > 0) {
long now = System.nanoTime();
if (start + 5000000000l < now) {
double complete = ((origLength - length) / (double) origLength) * 100.0;
out.println(complete + "% Complete");
out.flush();
start = System.nanoTime();
}
random.nextBytes(buf);
int len = (int) Math.min(length, bufSize);
long s = System.nanoTime();
outputStream.write(buf, 0, len);
long e = System.nanoTime();
time += (e - s);
length -= len;
}
return time;
}
}