// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.android.dexer;
import static com.google.common.base.Preconditions.checkArgument;
import static java.nio.charset.StandardCharsets.ISO_8859_1;
import static java.util.concurrent.Executors.newFixedThreadPool;
import com.android.dx.command.DxConsole;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.Weigher;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.android.Converters.ExistingPathConverter;
import com.google.devtools.build.android.Converters.PathConverter;
import com.google.devtools.build.android.dexer.Dexing.DexingKey;
import com.google.devtools.build.android.dexer.Dexing.DexingOptions;
import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest;
import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse;
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionsBase;
import com.google.devtools.common.options.OptionsParser;
import com.google.devtools.common.options.OptionsParser.OptionUsageRestrictions;
import com.google.devtools.common.options.OptionsParsingException;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import javax.annotation.Nullable;
/**
* Tool used by Bazel that converts a Jar file of .class files into a .zip file of .dex files,
* one per .class file, which we call a <i>dex archive</i>.
*/
class DexBuilder {
private static final long ONE_MEG = 1_000_000L;
/**
* Commandline options.
*/
public static class Options extends OptionsBase {
@Option(
name = "input_jar",
defaultValue = "null",
category = "input",
converter = ExistingPathConverter.class,
abbrev = 'i',
help = "Input file to read classes and jars from."
)
public Path inputJar;
@Option(
name = "output_zip",
defaultValue = "null",
category = "output",
converter = PathConverter.class,
abbrev = 'o',
help = "Output file to write."
)
public Path outputZip;
@Option(
name = "max_threads",
defaultValue = "8",
category = "misc",
help = "How many threads (besides the main thread) to use at most."
)
public int maxThreads;
@Option(
name = "persistent_worker",
defaultValue = "false",
optionUsageRestrictions = OptionUsageRestrictions.HIDDEN,
help = "Run as a Bazel persistent worker."
)
public boolean persistentWorker;
}
public static void main(String[] args) throws Exception {
if (args.length == 1 && args[0].startsWith("@")) {
args = Files.readAllLines(Paths.get(args[0].substring(1)), ISO_8859_1).toArray(new String[0]);
}
OptionsParser optionsParser =
OptionsParser.newOptionsParser(Options.class, DexingOptions.class);
optionsParser.parseAndExitUponError(args);
Options options = optionsParser.getOptions(Options.class);
if (options.persistentWorker) {
runPersistentWorker();
} else {
buildDexArchive(options, optionsParser.getOptions(DexingOptions.class));
}
}
@VisibleForTesting
static void buildDexArchive(Options options, DexingOptions dexingOptions)
throws Exception {
checkArgument(options.maxThreads > 0,
"--max_threads must be strictly positive, was: %s", options.maxThreads);
try (ZipFile in = new ZipFile(options.inputJar.toFile())) {
// Heuristic: use at most 1 thread per 1000 files in the input Jar
int threads = Math.min(options.maxThreads, in.size() / 1000 + 1);
ExecutorService executor = newFixedThreadPool(threads);
try (ZipOutputStream out = createZipOutputStream(options.outputZip)) {
produceDexArchive(in, out, executor, threads <= 1, dexingOptions, null);
} finally {
executor.shutdown();
}
}
// Use input's timestamp for output file so the output file is stable.
Files.setLastModifiedTime(options.outputZip, Files.getLastModifiedTime(options.inputJar));
}
/**
* Implements a persistent worker process for use with Bazel (see {@code WorkerSpawnStrategy}).
*/
private static void runPersistentWorker() throws IOException {
ExecutorService executor = newFixedThreadPool(Runtime.getRuntime().availableProcessors());
Cache<DexingKey, byte[]> dexCache = CacheBuilder.newBuilder()
// Use at most 200 MB for cache and leave at least 25 MB of heap space alone. For reference:
// .class & class.dex files are around 1-5 KB, so this fits ~30K-35K class-dex pairs.
.maximumWeight(Math.min(Runtime.getRuntime().maxMemory() - 25 * ONE_MEG, 200 * ONE_MEG))
.weigher(new Weigher<DexingKey, byte[]>() {
@Override
public int weigh(DexingKey key, byte[] value) {
return key.classfileContent().length + value.length;
}
})
.build();
try {
while (true) {
WorkRequest request = WorkRequest.parseDelimitedFrom(System.in);
if (request == null) {
return;
}
// Redirect dx's output so we can return it in response
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos, /*autoFlush*/ true);
DxConsole.out = DxConsole.err = ps;
// Make sure that we exit nonzero in case uncaught errors occur during processRequest.
int exitCode = 1;
try {
processRequest(executor, dexCache, request.getArgumentsList());
exitCode = 0; // success!
} catch (Exception e) {
// Deliberate catch-all so we can capture a stack trace.
// TODO(bazel-team): Consider canceling any outstanding futures created for this request
e.printStackTrace(ps);
} catch (Error e) {
e.printStackTrace();
e.printStackTrace(ps); // try capturing the error, may fail if out of memory
throw e; // rethrow to kill the worker
} finally {
// Try sending a response no matter what
String output;
try {
output = baos.toString();
} catch (Throwable t) { // most likely out of memory, so log with minimal memory needs
t.printStackTrace();
output = "check worker log for exceptions";
}
WorkResponse.newBuilder()
.setOutput(output)
.setExitCode(exitCode)
.build()
.writeDelimitedTo(System.out);
System.out.flush();
}
}
} finally {
executor.shutdown();
}
}
private static void processRequest(
ExecutorService executor, Cache<DexingKey, byte[]> dexCache, List<String> args)
throws OptionsParsingException, IOException, InterruptedException, ExecutionException {
OptionsParser optionsParser =
OptionsParser.newOptionsParser(Options.class, DexingOptions.class);
optionsParser.setAllowResidue(false);
optionsParser.parse(args);
Options options = optionsParser.getOptions(Options.class);
try (ZipFile in = new ZipFile(options.inputJar.toFile());
ZipOutputStream out = createZipOutputStream(options.outputZip)) {
produceDexArchive(
in,
out,
executor,
/*convertOnReaderThread*/ false,
optionsParser.getOptions(DexingOptions.class),
dexCache);
}
// Use input's timestamp for output file so the output file is stable.
Files.setLastModifiedTime(options.outputZip, Files.getLastModifiedTime(options.inputJar));
}
private static ZipOutputStream createZipOutputStream(Path path) throws IOException {
return new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(path)));
}
private static void produceDexArchive(
ZipFile in,
ZipOutputStream out,
ExecutorService executor,
boolean convertOnReaderThread,
DexingOptions dexingOptions,
@Nullable Cache<DexingKey, byte[]> dexCache)
throws InterruptedException, ExecutionException, IOException {
// If we only have one thread in executor, we give a "direct" executor to the stuffer, which
// will convert .class files to .dex inline on the same thread that reads the input jar.
// This is an optimization that makes sure we can start writing the output file below while
// the stuffer is still working its way through the input.
DexConversionEnqueuer enqueuer = new DexConversionEnqueuer(in,
convertOnReaderThread ? MoreExecutors.newDirectExecutorService() : executor,
new DexConverter(new Dexing(dexingOptions)),
dexCache);
Future<?> enqueuerTask = executor.submit(enqueuer);
while (true) {
// Wait for next future in the queue *and* for that future to finish. To guarantee
// deterministic output we just write out the files in the order they appear, which is
// the same order as in the input zip.
ZipEntryContent file = enqueuer.getFiles().take().get();
if (file == null) {
// "done" marker indicating no more files coming.
// Make sure enqueuer terminates normally (any wait should be minimal). This in
// particular surfaces any exceptions thrown in the enqueuer.
enqueuerTask.get();
break;
}
out.putNextEntry(file.getEntry());
out.write(file.getContent());
out.closeEntry();
}
}
private DexBuilder() {
}
}