/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.performance.tests;
import com.android.ddmlib.IDevice;
import com.android.ddmlib.MultiLineReceiver;
import com.android.ddmlib.NullOutputReceiver;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.Option.Importance;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.InputStreamSource;
import com.android.tradefed.result.LogDataType;
import com.android.tradefed.result.SnapshotInputStreamSource;
import com.android.tradefed.testtype.IDeviceTest;
import com.android.tradefed.testtype.IRemoteTest;
import junit.framework.Assert;
import junit.framework.TestCase;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
/**
* Runs the FIO benchmarks.
* <p>
* This test pushes the FIO executable to the device and runs several benchmarks. Running each
* benchmark consists of creating a config file, creating one or more data files, clearing the disk
* cache and then running FIO. The test runs a variety of different configurations including a
* simple benchmark with a single thread, a storage benchmark with 4 threads, a media server
* emulator, and a media scanner emulator.
* </p>
*/
public class FioBenchmarkTest implements IDeviceTest, IRemoteTest {
private static final String[] FIO_RESULT_FIELDS = {
"jobname", "groupid", "error",
// Read stats
"read-kb-io", "read-bandwidth", "read-runtime",
"read-slat-min", "read-slat-max", "read-slat-mean", "read-slat-stddev",
"read-clat-min", "read-clat-max", "read-clat-mean", "read-clat-stddev",
"read-bandwidth-min", "read-bandwidth-max", "read-bandwidth-percent", "read-bandwidth-mean",
"read-bandwidth-stddev",
// Write stats
"write-kb-io", "write-bandwidth", "write-runtime",
"write-slat-min", "write-slat-max", "write-slat-mean", "write-slat-stddev",
"write-clat-min", "write-clat-max", "write-clat-mean", "write-clat-stddev",
"write-bandwidth-min", "write-bandwidth-max", "write-bandwidth-percent",
"write-bandwidth-mean", "write-bandwidth-stddev",
// CPU stats
"cpu-user", "cpu-system", "cpu-context-switches", "cpu-major-page-faults",
"cpu-minor-page-faults",
// IO depth stats
"io-depth-1", "io-depth-2", "io-depth-4", "io-depth-8", "io-depth-16", "io-depth-32",
"io-depth-64",
// IO lat stats
"io-lat-2", "io-lat-4", "io-lat-10", "io-lat-20", "io-lat-50", "io-lat-100", "io-lat-250",
"io-lat-500", "io-lat-750", "io-lat-1000", "io-lat-2000"
};
private List<TestInfo> mTestCases = null;
/**
* Holds info about a job. The job translates into a job in the FIO config file. Contains the
* job name and a map from keys to values.
*/
private static class JobInfo {
public String mJobName = null;
public Map<String, String> mParameters = new HashMap<String, String>();
/**
* Gets the job as a string.
*
* @return a string of the job formatted for the config file.
*/
public String createJob() {
if (mJobName == null) {
return "";
}
StringBuilder sb = new StringBuilder();
sb.append(String.format("[%s]\n", mJobName));
for (Entry<String, String> parameter : mParameters.entrySet()) {
if (parameter.getValue() == null) {
sb.append(String.format("%s\n", parameter.getKey()));
} else {
sb.append(String.format("%s=%s\n", parameter.getKey(), parameter.getValue()));
}
}
return sb.toString();
}
}
/**
* Holds info about a file used in the benchmark. Because of limitations in FIO on Android,
* the file needs to be created before the tests are run. Contains the file name and size in kB.
*/
private static class TestFileInfo {
public String mFileName = null;
public int mSize = -1;
}
/**
* Holds info about the perf metric that are cared about for a given job.
*/
private static class PerfMetricInfo {
public enum ResultType {
STRING, INT, FLOAT, PERCENT;
String value(String input) {
switch(this) {
case STRING:
case INT:
case FLOAT:
return input;
case PERCENT:
if (input.length() < 2 || !input.endsWith("%")) {
return null;
}
try {
return String.format("%f", Double.parseDouble(
input.substring(0, input.length() - 1)) / 100);
} catch (NumberFormatException e) {
return null;
}
default: return null;
}
}
}
public String mJobName = null;
public String mFieldName = null;
public String mPostKey = null;
public ResultType mType = ResultType.STRING;
}
/**
* Holds the info associated with a test.
* <p>
* Contains the test name, key, a list of {@link JobInfo}, a set of {@link TestFileInfo},
* and a set of {@link PerfMetricInfo}.
* </p>
*/
private static class TestInfo {
public String mTestName = null;
public String mKey = null;
public List<JobInfo> mJobs = new LinkedList<JobInfo>();
public Set<TestFileInfo> mTestFiles = new HashSet<TestFileInfo>();
public Set<PerfMetricInfo> mPerfMetrics = new HashSet<PerfMetricInfo>();
/**
* Gets the config file.
*
* @return a string containing the contents of the config file needed to run the benchmark.
*/
private String createConfig() {
StringBuilder sb = new StringBuilder();
for (JobInfo job : mJobs) {
sb.append(String.format("%s\n", job.createJob()));
}
return sb.toString();
}
}
/**
* Parses the output of the FIO and allows the values to be looked up by job name and property.
*/
private static class FioParser extends MultiLineReceiver {
public Map<String, Map<String, String>> mResults =
new HashMap<String, Map<String, String>>();
/**
* Gets the result for a job and property, or null if the job or the property do not exist.
*
* @param job the name of the job.
* @param property the name of the property. See {@code FIO_RESULT_FIELDS}.
* @return the fio results for the job and property or null if it does not exist.
*/
public String getResult(String job, String property) {
if (!mResults.containsKey(job)) {
return null;
}
return mResults.get(job).get(property);
}
/**
* {@inheritDoc}
*/
@Override
public void processNewLines(String[] lines) {
for (String line : lines) {
CLog.d(line);
String[] fields = line.split(";");
if (fields.length < FIO_RESULT_FIELDS.length) {
continue;
}
Map<String, String> r = new HashMap<String, String>();
for (int i = 0; i < FIO_RESULT_FIELDS.length; i++) {
r.put(FIO_RESULT_FIELDS[i], fields[i]);
}
mResults.put(fields[0], r);
}
}
/**
* {@inheritDoc}
*/
@Override
public boolean isCancelled() {
return false;
}
}
ITestDevice mTestDevice = null;
private String mFioDir = null;
private String mFioBin = null;
private String mFioConfig = null;
@Option(name="fio-location", description="The path to the precompiled FIO executable.",
importance=Importance.ALWAYS)
private String mFioLocation = null;
@Option(name="tmp-dir", description="The directory used for interal benchmarks.")
private String mTmpDir = "/data/tmp/fio";
@Option(name="internal-test-dir", description="The directory used for interal benchmarks.")
private String mInternalTestDir = "/data/fio/data";
@Option(name="media-test-dir", description="The directory used for media benchmarks.")
private String mMediaTestDir = "${EXTERNAL_STORAGE}/fio";
@Option(name="external-test-dir", description="The directory used for external benchmarks.")
private String mExternalTestDir = "${EXTERNAL_STORAGE}/fio";
@Option(name="collect-yaffs-logs", description="Collect yaffs logs before and after tests")
private Boolean mCollectYaffsLogs = false;
@Option(name="run-simple-internal-test", description="Run the simple internal benchmark.")
private Boolean mRunSimpleInternalTest = true;
@Option(name="simple-internal-file-size",
description="The file size of the simple internal benchmark in MB.")
private int mSimpleInternalFileSize = 256;
@Option(name="run-simple-external-test", description="Run the simple external benchmark.")
private Boolean mRunSimpleExternalTest = false;
@Option(name="simple-external-file-size",
description="The file size of the simple external benchmark in MB.")
private int mSimpleExternalFileSize = 256;
@Option(name="run-storage-internal-test", description="Run the storage internal benchmark.")
private Boolean mRunStorageInternalTest = true;
@Option(name="storage-internal-file-size",
description="The file size of the storage internal benchmark in MB.")
private int mStorageInternalFileSize = 256;
@Option(name="storage-internal-job-count",
description="The number of jobs for the storage internal benchmark.")
private int mStorageInternalJobCount = 4;
@Option(name="run-storage-external-test", description="Run the storage external benchmark.")
private Boolean mRunStorageExternalTest = false;
@Option(name="storage-external-file-size",
description="The file size of the storage external benchmark in MB.")
private int mStorageExternalFileSize = 256;
@Option(name="storage-external-job-count",
description="The number of jobs for the storage external benchmark.")
private int mStorageExternalJobCount = 4;
@Option(name="run-media-server-test", description="Run the media server benchmark.")
private Boolean mRunMediaServerTest = false;
@Option(name="media-server-duration",
description="The duration of the media server benchmark in secs.")
private long mMediaServerDuration = 30;
@Option(name="media-server-media-file-size",
description="The media file size of the media server benchmark in MB.")
private int mMediaServerMediaFileSize = 256;
@Option(name="media-server-worker-file-size",
description="The worker file size of the media server benchmark in MB.")
private int mMediaServerWorkerFileSize = 256;
@Option(name="media-server-worker-job-count",
description="The number of worker jobs for the media server benchmark.")
private int mMediaServerWorkerJobCount = 4;
@Option(name="run-media-scanner-test", description="Run the media scanner benchmark.")
private Boolean mRunMediaScannerTest = false;
@Option(name="media-scanner-media-file-size",
description="The media file size of the media scanner benchmark in kB.")
private int mMediaScannerMediaFileSize = 8;
@Option(name="media-scanner-media-file-count",
description="The number of media files to scan.")
private int mMediaScannerMediaFileCount = 256;
@Option(name="media-scanner-worker-file-size",
description="The worker file size of the media scanner benchmark in MB.")
private int mMediaScannerWorkerFileSize = 256;
@Option(name="media-scanner-worker-job-count",
description="The number of worker jobs for the media server benchmark.")
private int mMediaScannerWorkerJobCount = 4;
/**
* Sets up all the benchmarks.
*/
private void setupTests() {
if (mTestCases != null) {
// assume already set up
return;
}
mTestCases = new LinkedList<TestInfo>();
if (mRunSimpleInternalTest) {
addSimpleTest("read", true);
addSimpleTest("write", true);
addSimpleTest("randread", true);
addSimpleTest("randwrite", true);
}
if (mRunSimpleExternalTest) {
addSimpleTest("read", false);
addSimpleTest("write", false);
addSimpleTest("randread", false);
addSimpleTest("randwrite", false);
}
if (mRunStorageInternalTest) {
addStorageTest("read", true);
addStorageTest("write", true);
addStorageTest("randread", true);
addStorageTest("randwrite", true);
}
if (mRunStorageExternalTest) {
addStorageTest("read", false);
addStorageTest("write", false);
addStorageTest("randread", false);
addStorageTest("randwrite", false);
}
if (mRunMediaServerTest) {
addMediaServerTest("read");
addMediaServerTest("write");
}
if (mRunMediaScannerTest) {
addMediaScannerTest();
}
}
/**
* Sets up the simple FIO benchmark.
* <p>
* The test consists of a single process reading or writing to a file.
* </p>
*
* @param rw the type of IO pattern. One of {@code read}, {@code write}, {@code randread}, or
* {@code randwrite}.
* @param internal whether the test should be run on the internal (/data) partition or the
* external partition.
*/
private void addSimpleTest(String rw, boolean internal) {
String fileName = "testfile";
String jobName = "job";
String directory;
int fileSize;
TestInfo t = new TestInfo();
if (internal) {
t.mTestName = String.format("SimpleBenchmark-int-%s", rw);
t.mKey = "fio_simple_int_benchmark";
directory = mInternalTestDir;
fileSize = mSimpleInternalFileSize;
} else {
t.mTestName = String.format("SimpleBenchmark-ext-%s", rw);
t.mKey = "fio_simple_ext_benchmark";
directory = mExternalTestDir;
fileSize = mSimpleExternalFileSize;
}
TestFileInfo f = new TestFileInfo();
f.mFileName = new File(directory, fileName).getAbsolutePath();
f.mSize = fileSize * 1024; // fileSize is in MB but we want it in kB.
t.mTestFiles.add(f);
JobInfo j = new JobInfo();
j.mJobName = jobName;
j.mParameters.put("directory", directory);
j.mParameters.put("filename", fileName);
j.mParameters.put("fsync", "1024");
j.mParameters.put("ioengine", "sync");
j.mParameters.put("rw", rw);
j.mParameters.put("size", String.format("%dM",fileSize));
t.mJobs.add(j);
PerfMetricInfo m = new PerfMetricInfo();
m.mJobName = jobName;
m.mPostKey = String.format("%s_bandwidth", rw);
m.mType = PerfMetricInfo.ResultType.FLOAT;
if (rw.endsWith("read")) {
m.mFieldName = "read-bandwidth-mean";
} else if (rw.endsWith("write")) {
m.mFieldName = "write-bandwidth-mean";
}
t.mPerfMetrics.add(m);
mTestCases.add(t);
}
/**
* Sets up the storage FIO benchmark.
* <p>
* The test consists of several processes reading or writing to a file.
* </p>
*
* @param rw the type of IO pattern. One of {@code read}, {@code write}, {@code randread}, or
* {@code randwrite}.
* @param internal whether the test should be run on the internal (/data) partition or the
* external partition.
*/
private void addStorageTest(String rw, boolean internal) {
String fileName = "testfile";
String jobName = "workers";
String directory;
int fileSize;
int jobCount;
TestInfo t = new TestInfo();
if (internal) {
t.mTestName = String.format("StorageBenchmark-int-%s", rw);
t.mKey = "fio_storage_int_benchmark";
directory = mInternalTestDir;
fileSize = mStorageInternalFileSize;
jobCount = mStorageInternalJobCount;
} else {
t.mTestName = String.format("StorageBenchmark-ext-%s", rw);
t.mKey = "fio_storage_ext_benchmark";
directory = mExternalTestDir;
fileSize = mStorageExternalFileSize;
jobCount = mStorageExternalJobCount;
}
TestFileInfo f = new TestFileInfo();
f.mFileName = new File(directory, fileName).getAbsolutePath();
f.mSize = fileSize * 1024; // fileSize is in MB but we want it in kB.
t.mTestFiles.add(f);
JobInfo j = new JobInfo();
j.mJobName = jobName;
j.mParameters.put("directory", directory);
j.mParameters.put("filename", fileName);
j.mParameters.put("fsync", "1024");
j.mParameters.put("group_reporting", null);
j.mParameters.put("ioengine", "sync");
j.mParameters.put("new_group", null);
j.mParameters.put("numjobs", String.format("%d", jobCount));
j.mParameters.put("rw", rw);
j.mParameters.put("size", String.format("%dM",fileSize));
t.mJobs.add(j);
PerfMetricInfo m = new PerfMetricInfo();
m.mJobName = jobName;
m.mPostKey = String.format("%s_bandwidth", rw);
m.mType = PerfMetricInfo.ResultType.FLOAT;
if (rw.endsWith("read")) {
m.mFieldName = "read-bandwidth-mean";
} else if (rw.endsWith("write")) {
m.mFieldName = "write-bandwidth-mean";
}
t.mPerfMetrics.add(m);
m = new PerfMetricInfo();
m.mJobName = jobName;
m.mPostKey = String.format("%s_latency", rw);
m.mType = PerfMetricInfo.ResultType.FLOAT;
if (rw.endsWith("read")) {
m.mFieldName = "read-clat-mean";
} else if (rw.endsWith("write")) {
m.mFieldName = "write-clat-mean";
}
t.mPerfMetrics.add(m);
mTestCases.add(t);
}
/**
* Sets up the media server benchmark.
* <p>
* The test consists of a single process at a higher priority reading or writing to a file
* while several other worker processes read and write to a different file.
* </p>
*
* @param rw the type of IO pattern. One of {@code read}, {@code write}
*/
private void addMediaServerTest(String rw) {
String mediaJob = "media-server";
String mediaFile = "mediafile";
String workerJob = "workers";
String workerFile = "workerfile";
TestInfo t = new TestInfo();
t.mTestName = String.format("MediaServerBenchmark-%s", rw);
t.mKey = "fio_media_server_benchmark";
TestFileInfo f = new TestFileInfo();
f.mFileName = new File(mMediaTestDir, mediaFile).getAbsolutePath();
f.mSize = mMediaServerMediaFileSize * 1024; // File size is in MB but we want it in kB.
t.mTestFiles.add(f);
f = new TestFileInfo();
f.mFileName = new File(mMediaTestDir, workerFile).getAbsolutePath();
f.mSize = mMediaServerWorkerFileSize * 1024; // File size is in MB but we want it in kB.
t.mTestFiles.add(f);
JobInfo j = new JobInfo();
j.mJobName = "global";
j.mParameters.put("directory", mMediaTestDir);
j.mParameters.put("fsync", "1024");
j.mParameters.put("ioengine", "sync");
j.mParameters.put("runtime", String.format("%d", mMediaServerDuration));
j.mParameters.put("time_based", null);
t.mJobs.add(j);
j = new JobInfo();
j.mJobName = mediaJob;
j.mParameters.put("filename", mediaFile);
j.mParameters.put("iodepth", "32");
j.mParameters.put("nice", "-16");
j.mParameters.put("rate", "6m");
j.mParameters.put("rw", rw);
j.mParameters.put("size", String.format("%dM", mMediaServerMediaFileSize));
t.mJobs.add(j);
j = new JobInfo();
j.mJobName = workerJob;
j.mParameters.put("filename", workerFile);
j.mParameters.put("group_reporting", null);
j.mParameters.put("new_group", null);
j.mParameters.put("nice", "0");
j.mParameters.put("numjobs", String.format("%d", mMediaServerWorkerJobCount));
j.mParameters.put("rw", "randrw");
j.mParameters.put("size", String.format("%dM", mMediaServerWorkerFileSize));
t.mJobs.add(j);
PerfMetricInfo m = new PerfMetricInfo();
m.mJobName = mediaJob;
m.mPostKey = String.format("%s_media_bandwidth", rw);
m.mType = PerfMetricInfo.ResultType.FLOAT;
if (rw.endsWith("read")) {
m.mFieldName = "read-bandwidth-mean";
} else if (rw.endsWith("write")) {
m.mFieldName = "write-bandwidth-mean";
}
t.mPerfMetrics.add(m);
m = new PerfMetricInfo();
m.mJobName = mediaJob;
m.mPostKey = String.format("%s_media_latency", rw);
m.mType = PerfMetricInfo.ResultType.FLOAT;
if (rw.endsWith("read")) {
m.mFieldName = "read-clat-mean";
} else if (rw.endsWith("write")) {
m.mFieldName = "write-clat-mean";
}
t.mPerfMetrics.add(m);
m = new PerfMetricInfo();
m.mJobName = workerJob;
m.mPostKey = String.format("%s_workers_read_bandwidth", rw);
m.mFieldName = "read-bandwidth-mean";
m.mType = PerfMetricInfo.ResultType.FLOAT;
t.mPerfMetrics.add(m);
m = new PerfMetricInfo();
m.mJobName = workerJob;
m.mPostKey = String.format("%s_workers_write_bandwidth", rw);
m.mFieldName = "write-bandwidth-mean";
m.mType = PerfMetricInfo.ResultType.FLOAT;
t.mPerfMetrics.add(m);
mTestCases.add(t);
}
/**
* Sets up the media scanner benchmark.
* <p>
* The test consists of a single process reading several small files while several other worker
* processes read and write to a different file.
* </p>
*/
private void addMediaScannerTest() {
String mediaJob = "media-server";
String mediaFile = "mediafile.%d";
String workerJob = "workers";
String workerFile = "workerfile";
TestInfo t = new TestInfo();
t.mTestName = "MediaScannerBenchmark";
t.mKey = "fio_media_scanner_benchmark";
TestFileInfo f;
for (int i = 0; i < mMediaScannerMediaFileCount; i++) {
f = new TestFileInfo();
f.mFileName = new File(mMediaTestDir, String.format(mediaFile, i)).getAbsolutePath();
f.mSize = mMediaScannerMediaFileSize; // File size is already in kB so do nothing.
t.mTestFiles.add(f);
}
f = new TestFileInfo();
f.mFileName = new File(mMediaTestDir, workerFile).getAbsolutePath();
f.mSize = mMediaScannerWorkerFileSize * 1024; // File size is in MB but we want it in kB.
t.mTestFiles.add(f);
JobInfo j = new JobInfo();
j.mJobName = "global";
j.mParameters.put("directory", mMediaTestDir);
j.mParameters.put("fsync", "1024");
j.mParameters.put("ioengine", "sync");
t.mJobs.add(j);
j = new JobInfo();
j.mJobName = mediaJob;
StringBuilder fileNames = new StringBuilder();
fileNames.append(String.format(mediaFile, 0));
for (int i = 1; i < mMediaScannerMediaFileCount; i++) {
fileNames.append(String.format(":%s", String.format(mediaFile, i)));
}
j.mParameters.put("filename", fileNames.toString());
j.mParameters.put("exitall", null);
j.mParameters.put("openfiles", "4");
j.mParameters.put("rw", "read");
t.mJobs.add(j);
j = new JobInfo();
j.mJobName = workerJob;
j.mParameters.put("filename", workerFile);
j.mParameters.put("group_reporting", null);
j.mParameters.put("new_group", null);
j.mParameters.put("numjobs", String.format("%d", mMediaScannerWorkerJobCount));
j.mParameters.put("rw", "randrw");
j.mParameters.put("size", String.format("%dM", mMediaScannerWorkerFileSize));
t.mJobs.add(j);
PerfMetricInfo m = new PerfMetricInfo();
m.mJobName = mediaJob;
m.mPostKey = "media_bandwidth";
m.mFieldName = "read-bandwidth-mean";
m.mType = PerfMetricInfo.ResultType.FLOAT;
t.mPerfMetrics.add(m);
m = new PerfMetricInfo();
m.mJobName = mediaJob;
m.mPostKey = "media_latency";
m.mFieldName = "read-clat-mean";
m.mType = PerfMetricInfo.ResultType.FLOAT;
t.mPerfMetrics.add(m);
m = new PerfMetricInfo();
m.mJobName = workerJob;
m.mPostKey = "workers_read_bandwidth";
m.mFieldName = "read-bandwidth-mean";
m.mType = PerfMetricInfo.ResultType.FLOAT;
t.mPerfMetrics.add(m);
m = new PerfMetricInfo();
m.mJobName = workerJob;
m.mPostKey = "workers_write_bandwidth";
m.mFieldName = "write-bandwidth-mean";
m.mType = PerfMetricInfo.ResultType.FLOAT;
t.mPerfMetrics.add(m);
mTestCases.add(t);
}
/**
* Creates the directories needed to run FIO, pushes the FIO executable to the device, and
* stops the runtime.
*
* @throws DeviceNotAvailableException if the device is not available.
*/
private void setupDevice() throws DeviceNotAvailableException {
mTestDevice.executeShellCommand("stop");
mTestDevice.executeShellCommand(String.format("mkdir -p %s", mFioDir));
mTestDevice.executeShellCommand(String.format("mkdir -p %s", mTmpDir));
mTestDevice.executeShellCommand(String.format("mkdir -p %s", mInternalTestDir));
mTestDevice.executeShellCommand(String.format("mkdir -p %s", mMediaTestDir));
if (mExternalTestDir != null) {
mTestDevice.executeShellCommand(String.format("mkdir -p %s", mExternalTestDir));
}
mTestDevice.pushFile(new File(mFioLocation), mFioBin);
mTestDevice.executeShellCommand(String.format("chmod 755 %s", mFioBin));
}
/**
* Reverses the actions of {@link #setDevice(ITestDevice)}.
*
* @throws DeviceNotAvailableException If the device is not available.
*/
private void cleanupDevice() throws DeviceNotAvailableException {
if (mExternalTestDir != null) {
mTestDevice.executeShellCommand(String.format("rm -r %s", mExternalTestDir));
}
mTestDevice.executeShellCommand(String.format("rm -r %s", mMediaTestDir));
mTestDevice.executeShellCommand(String.format("rm -r %s", mInternalTestDir));
mTestDevice.executeShellCommand(String.format("rm -r %s", mTmpDir));
mTestDevice.executeShellCommand(String.format("rm -r %s", mFioDir));
mTestDevice.executeShellCommand("start");
}
/**
* Runs a single test, including creating the test files, clearing the cache, collecting before
* and after files, running the benchmark, and reporting the results.
*
* @param test the benchmark.
* @param listener the ITestInvocationListener
* @throws DeviceNotAvailableException if the device is not available.
*/
private void runTest(TestInfo test, ITestInvocationListener listener)
throws DeviceNotAvailableException {
CLog.i("Running %s benchmark", test.mTestName);
mTestDevice.executeShellCommand(String.format("rm -r %s/*", mTmpDir));
mTestDevice.executeShellCommand(String.format("rm -r %s/*", mInternalTestDir));
mTestDevice.executeShellCommand(String.format("rm -r %s/*", mMediaTestDir));
if (mExternalTestDir != null) {
mTestDevice.executeShellCommand(String.format("rm -r %s/*", mExternalTestDir));
}
for (TestFileInfo file : test.mTestFiles) {
CLog.v("Creating file: %s, size: %dkB", file.mFileName, file.mSize);
String cmd = String.format("dd if=/dev/urandom of=%s bs=1024 count=%d", file.mFileName,
file.mSize);
int timeout = file.mSize * 2 * 1000; // Timeout is 2 seconds per kB.
mTestDevice.executeShellCommand(cmd, new NullOutputReceiver(), timeout, 2);
}
CLog.i("Creating config");
CLog.d("Config file:\n%s", test.createConfig());
mTestDevice.pushString(test.createConfig(), mFioConfig);
CLog.i("Dropping cache");
mTestDevice.executeShellCommand("echo 3 > /proc/sys/vm/drop_caches");
collectLogs(test, listener, "before");
CLog.i("Running test");
FioParser output = new FioParser();
// Run FIO with a timeout of 1 hour.
mTestDevice.executeShellCommand(String.format("%s --minimal %s", mFioBin, mFioConfig),
output, 60 * 60 * 1000, 2);
collectLogs(test, listener, "after");
// Report metrics
Map<String, String> metrics = new HashMap<String, String>();
listener.testRunStarted(test.mKey, 0);
for (PerfMetricInfo m : test.mPerfMetrics) {
if (!output.mResults.containsKey(m.mJobName)) {
CLog.w("Job name %s was not found in the results", m.mJobName);
continue;
}
String value = output.getResult(m.mJobName, m.mFieldName);
if (value != null) {
metrics.put(m.mPostKey, m.mType.value(value));
} else {
CLog.w("%s was not in results for the job %s", m.mFieldName, m.mJobName);
}
}
CLog.d("About to report metrics to %s: %s", test.mKey, metrics);
listener.testRunEnded(0, metrics);
}
private void collectLogs(TestInfo testInfo, ITestInvocationListener listener,
String descriptor) throws DeviceNotAvailableException {
if (mCollectYaffsLogs && mTestDevice.doesFileExist("/proc/yaffs")) {
logFile("/proc/yaffs", String.format("%s-yaffs-%s", testInfo.mTestName, descriptor),
mTestDevice, listener);
}
}
private void logFile(String remoteFileName, String localFileName, ITestDevice testDevice,
ITestInvocationListener listener) throws DeviceNotAvailableException {
File outputFile = null;
InputStreamSource outputSource = null;
try {
outputFile = mTestDevice.pullFile(remoteFileName);
if (outputFile != null) {
CLog.d("Sending %d byte file %s to logosphere!", outputFile.length(), outputFile);
outputSource = new SnapshotInputStreamSource(new FileInputStream(outputFile));
listener.testLog(localFileName, LogDataType.TEXT, outputSource);
}
} catch (IOException e) {
CLog.e("Got an IO Exception: %s", e);
} finally {
if (outputSource != null) {
outputSource.cancel();
}
if (outputFile != null) {
outputFile.delete();
}
}
}
/**
* {@inheritDoc}
*/
@Override
public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
Assert.assertNotNull(mTestDevice);
mFioDir = new File(mTestDevice.getMountPoint(IDevice.MNT_DATA), "fio").getAbsolutePath();
mFioBin = new File(mFioDir, "fio").getAbsolutePath();
mFioConfig = new File(mFioDir, "config.fio").getAbsolutePath();
setupTests();
setupDevice();
for (TestInfo test : mTestCases) {
runTest(test, listener);
}
cleanupDevice();
}
/**
* {@inheritDoc}
*/
@Override
public void setDevice(ITestDevice device) {
mTestDevice = device;
}
/**
* {@inheritDoc}
*/
@Override
public ITestDevice getDevice() {
return mTestDevice;
}
/**
* A meta-test to ensure that the bits of FioBenchmarkTest are working properly.
*/
public static class MetaTest extends TestCase {
/**
* Test that {@link JobInfo#createJob()} properly formats a job.
*/
public void testCreateJob() {
JobInfo j = new JobInfo();
assertEquals("", j.createJob());
j.mJobName = "job";
assertEquals("[job]\n", j.createJob());
j.mParameters.put("param1", null);
j.mParameters.put("param2", "value");
String[] lines = j.createJob().split("\n");
assertEquals(3, lines.length);
assertEquals("[job]", lines[0]);
Set<String> params = new HashSet<String>(2);
params.add(lines[1]);
params.add(lines[2]);
assertTrue(params.contains("param1"));
assertTrue(params.contains("param2=value"));
}
/**
* Test that {@link TestInfo#createConfig()} properly formats a config.
*/
public void testCreateConfig() {
TestInfo t = new TestInfo();
JobInfo j = new JobInfo();
j.mJobName = "job1";
j.mParameters.put("param1", "value1");
t.mJobs.add(j);
j = new JobInfo();
j.mJobName = "job2";
j.mParameters.put("param2", "value2");
t.mJobs.add(j);
j = new JobInfo();
j.mJobName = "job3";
j.mParameters.put("param3", "value3");
t.mJobs.add(j);
assertEquals("[job1]\nparam1=value1\n\n" +
"[job2]\nparam2=value2\n\n" +
"[job3]\nparam3=value3\n\n", t.createConfig());
}
/**
* Test that output lines are parsed correctly by the FioParser, invalid lines are ignored,
* and that the various fields are accessible with
* {@link FioParser#getResult(String, String)}.
*/
public void testFioParser() {
String[] lines = new String[4];
// We build the lines up as follows (assuming FIO_RESULTS_FIELDS.length == 58):
// 0;3;6;...;171
// 1;4;7;...;172
// 2;5;8;...;173
for (int i = 0; i < 3; i++) {
StringBuilder sb = new StringBuilder();
sb.append(i);
for (int j = 1; j < FIO_RESULT_FIELDS.length; j++) {
sb.append(";");
sb.append(j * 3 + i);
}
lines[i] = sb.toString();
}
// A line may have an optional description on the end which we don't care about, make
// sure it still parses.
lines[2] = lines[2] += ";description";
// Make sure an invalid output line does not parse.
lines[3] = "invalid";
FioParser p = new FioParser();
p.processNewLines(lines);
for (int i = 0; i < 3; i++) {
for (int j = 0; j < FIO_RESULT_FIELDS.length; j++) {
assertEquals(String.format("job=%d, field=%s", i, FIO_RESULT_FIELDS[j]),
String.format("%d", j * 3 + i),
p.getResult(String.format("%d", i), FIO_RESULT_FIELDS[j]));
}
}
assertNull(p.getResult("missing", "jobname"));
assertNull(p.getResult("invalid", "jobname"));
assertNull(p.getResult("0", "missing"));
}
/**
* Test that {@link PerfMetricInfo.ResultType#value(String)} correctly transforms strings
* based on the result type.
*/
public void testResultTypeValue() {
assertEquals("test", PerfMetricInfo.ResultType.STRING.value("test"));
assertEquals("1", PerfMetricInfo.ResultType.INT.value("1"));
assertEquals("3.14159", PerfMetricInfo.ResultType.FLOAT.value("3.14159"));
assertEquals(String.format("%f", 0.34567),
PerfMetricInfo.ResultType.PERCENT.value("34.567%"));
assertNull(PerfMetricInfo.ResultType.PERCENT.value(""));
assertNull(PerfMetricInfo.ResultType.PERCENT.value("34.567"));
assertNull(PerfMetricInfo.ResultType.PERCENT.value("test%"));
}
}
}