// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.android.dexer;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.util.concurrent.Futures.immediateFuture;
import com.google.common.cache.Cache;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.android.dexer.Dexing.DexingKey;
import java.io.IOException;
import java.io.InputStream;
import java.util.Enumeration;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import javax.annotation.Nullable;
/**
* Worker that reads an input Jar file and creates futures to convert .class to .dex files while
* leaving other files in the Jar unchanged. Converted files appear in {@link #getFiles()}.
* Because the queue of files is size-limited, this {@link Callable} must not be invoked on the
* main thread to avoid deadlocking.
*
* <p>Note on the name: this callable enqueues futures to convert .class files into a thread pool;
* it doesn't return a value itself other than successful termination or an exception during input
* file reading.
*/
class DexConversionEnqueuer implements Callable<Void> {
private static final byte[] EMPTY = new byte[0];
private final ZipFile in;
private final DexConverter dexer;
private final ExecutorService executor;
@Nullable private final Cache<DexingKey, byte[]> dexCache;
/** Converted content of the input file. See {@link #getFiles()} for more details. */
// Rate-limit to 30000 files in flight at once, which is about what we've tested. Theoretically,
// an unbounded queue can lead to OutOfMemoryErrors, and any limit is helpful so that one can
// set -Xmx to handle even large files. The "downside" is that this callable can theoretically
// block trying to add more elements to the queue, wasting the thread during that time.
private final BlockingQueue<Future<ZipEntryContent>> files = new ArrayBlockingQueue<>(30000);
public DexConversionEnqueuer(ZipFile in, ExecutorService executor, DexConverter dexer,
@Nullable Cache<DexingKey, byte[]> dexCache) {
this.in = in;
this.executor = executor;
this.dexer = dexer;
this.dexCache = dexCache;
}
@Override
public Void call() throws InterruptedException, IOException {
try {
Enumeration<? extends ZipEntry> entries = in.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
// Since these entries come from an existing zip file, they should always know their size
// (meaning, never return -1). We also can't handle files that don't fit into a byte array.
checkArgument(entry.getSize() >= 0, "Cannot process entry with unknown size: %s", entry);
checkArgument(entry.getSize() <= Integer.MAX_VALUE, "Entry too large: %s", entry);
byte[] content;
if (entry.getSize() == 0L) {
content = EMPTY; // this in particular covers directory entries
} else {
try (InputStream entryStream = in.getInputStream(entry)) {
// Read all the content at once, which avoids temporary arrays and extra array copies
content = new byte[(int) entry.getSize()];
ByteStreams.readFully(entryStream, content); // throws if file is smaller than expected
checkState(entryStream.read() == -1,
"Too many bytes in jar entry %s, expected %s", entry, entry.getSize());
}
}
if (!entry.isDirectory() && entry.getName().endsWith(".class")) {
files.put(toDex(entry, content));
} else {
// Copy other files and directory entries
if (entry.getCompressedSize() != 0) {
entry.setCompressedSize(-1L); // We may compress differently from source Zip
}
files.put(immediateFuture(new ZipEntryContent(entry, content)));
}
}
} finally {
// Use try-finally to make absolutely sure we do this, otherwise the reader might deadlock
files.put(immediateFuture((ZipEntryContent) null)); // "end of stream" marker
}
return null;
}
private Future<ZipEntryContent> toDex(ZipEntry entry, byte[] content) {
byte[] cached = dexCache != null ? dexCache.getIfPresent(dexer.getDexingKey(content)) : null;
return cached != null
? immediateFuture(newDexEntry(entry, cached))
: executor.submit(new ClassToDex(entry, content, dexer, dexCache));
}
/**
* Converted .dex files as well as (unchanged) resources in the order they appear in {@link #in
* the input zip file}. For simplicity we use a separate future for each file, followed by a
* future returning {@code null} to indicate that the input zip file is exhausted. To achieve
* determinism, the consumer of this queue should write the content of each file in the order
* they appear in this queue. Note that no files will appear in this queue until this callable is
* {@link #call invoked}, typically by submitting it to an {@link ExecutorService}. Once a
* future returning {@code null} appears in the queue, no more elements will follow, so the
* consumer should be a single-threaded loop that terminates on this {@code null} sentinel.
*/
public BlockingQueue<Future<ZipEntryContent>> getFiles() {
return files;
}
private static ZipEntryContent newDexEntry(ZipEntry classfile, byte[] dexed) {
return new ZipEntryContent(withFileName(classfile, classfile.getName() + ".dex"), dexed);
}
private static ZipEntry withFileName(ZipEntry orig, String filename) {
ZipEntry result = new ZipEntry(filename);
result.setTime(orig.getTime());
return result;
}
/**
* Worker to convert a {@code byte[]} representing a .class file into a {@code byte[]}
* representing a .dex file.
*/
private static class ClassToDex implements Callable<ZipEntryContent> {
private final ZipEntry entry;
private final byte[] content;
private final DexConverter dexer;
@Nullable private final Cache<DexingKey, byte[]> dexCache;
public ClassToDex(ZipEntry entry, byte[] content, DexConverter dexer,
@Nullable Cache<DexingKey, byte[]> dexCache) {
this.entry = entry;
this.content = content;
this.dexer = dexer;
this.dexCache = dexCache;
}
@Override
public ZipEntryContent call() throws Exception {
byte[] dexed = DexFiles.encode(dexer.toDexFile(content, entry.getName()));
if (dexCache != null) {
dexCache.put(dexer.getDexingKey(content), dexed);
}
// Use .class.dex suffix expected by SplitZip
return newDexEntry(entry, dexed);
}
}
}