// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.singlejar;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy;
import com.google.devtools.build.singlejar.ZipEntryFilter.StrategyCallback;
import com.google.devtools.build.zip.ExtraData;
import com.google.devtools.build.zip.ExtraDataList;
import com.google.devtools.build.zip.ZipFileEntry;
import com.google.devtools.build.zip.ZipFileEntry.Compression;
import com.google.devtools.build.zip.ZipReader;
import com.google.devtools.build.zip.ZipUtil;
import com.google.devtools.build.zip.ZipWriter;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.CRC32;
import java.util.zip.Deflater;
import java.util.zip.DeflaterInputStream;
import java.util.zip.Inflater;
import java.util.zip.InflaterInputStream;
import javax.annotation.Nullable;
/**
* An object that combines multiple ZIP files into a single file. It only
* supports a subset of the ZIP format, specifically:
* <ul>
* <li>It only supports STORE and DEFLATE storage methods.</li>
* <li>It only supports 32-bit ZIP files.</li>
* </ul>
*
* <p>These restrictions are also present in the JDK implementations
* {@link java.util.jar.JarInputStream}, {@link java.util.zip.ZipInputStream},
* though they are not documented there.
*
* <p>IMPORTANT NOTE: Callers must call {@link #finish()} or {@link #close()}
* at the end of processing to ensure that the output buffers are flushed and
* the ZIP file is complete.
*
* <p>This class performs only rudimentary data checking. If the input files
* are damaged, the output will likely also be damaged.
*
* <p>Also see:
* <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP format</a>
*/
public class ZipCombiner implements AutoCloseable {
public static final Date DOS_EPOCH = new Date(ZipUtil.DOS_EPOCH);
/**
* Whether to compress or decompress entries.
*/
public enum OutputMode {
/**
* Output entries using any method.
*/
DONT_CARE,
/**
* Output all entries using DEFLATE method, except directory entries. It is always more
* efficient to store directory entries uncompressed.
*/
FORCE_DEFLATE,
/**
* Output all entries using STORED method.
*/
FORCE_STORED,
}
/**
* The type of action to take for a ZIP file entry.
*/
private enum ActionType {
/**
* Skip the entry.
*/
SKIP,
/**
* Copy the entry.
*/
COPY,
/**
* Rename the entry.
*/
RENAME,
/**
* Merge the entry.
*/
MERGE;
}
/**
* Encapsulates the action to take for a ZIP file entry along with optional details specific to
* the action type. The minimum requirements per type are:
* <ul>
* <li>SKIP: none.</li>
* <li>COPY: none.</li>
* <li>RENAME: newName.</li>
* <li>MERGE: strategy, mergeBuffer.</li>
* </ul>
*
* <p>An action can be easily changed from one type to another by using
* {@link EntryAction#EntryAction(ActionType, EntryAction)}.
*/
private static final class EntryAction {
private final ActionType type;
@Nullable private final Date date;
@Nullable private final String newName;
@Nullable private final CustomMergeStrategy strategy;
@Nullable private final ByteArrayOutputStream mergeBuffer;
/**
* Create an action of the specified type with no extra details.
*/
public EntryAction(ActionType type) {
this(type, null, null, null, null);
}
/**
* Create a duplicate action with a different {@link ActionType}.
*/
public EntryAction(ActionType type, EntryAction action) {
this(type, action.getDate(), action.getNewName(), action.getStrategy(),
action.getMergeBuffer());
}
/**
* Create an action of the specified type and details.
*
* @param type the type of action
* @param date the custom date to set on the entry
* @param newName the custom name to create the entry as
* @param strategy the {@link CustomMergeStrategy} to use for merging this entry
* @param mergeBuffer the output stream to use for merge results
*/
public EntryAction(ActionType type, Date date, String newName, CustomMergeStrategy strategy,
ByteArrayOutputStream mergeBuffer) {
checkArgument(type != ActionType.RENAME || newName != null,
"NewName must not be null if the ActionType is RENAME.");
checkArgument(type != ActionType.MERGE || strategy != null,
"Strategy must not be null if the ActionType is MERGE.");
checkArgument(type != ActionType.MERGE || mergeBuffer != null,
"MergeBuffer must not be null if the ActionType is MERGE.");
this.type = type;
this.date = date;
this.newName = newName;
this.strategy = strategy;
this.mergeBuffer = mergeBuffer;
}
/** Returns the type. */
public ActionType getType() {
return type;
}
/** Returns the date. */
public Date getDate() {
return date;
}
/** Returns the new name. */
public String getNewName() {
return newName;
}
/** Returns the strategy. */
public CustomMergeStrategy getStrategy() {
return strategy;
}
/** Returns the mergeBuffer. */
public ByteArrayOutputStream getMergeBuffer() {
return mergeBuffer;
}
}
private final class FilterCallback implements StrategyCallback {
private String filename;
private final AtomicBoolean called = new AtomicBoolean();
public void resetForFile(String filename) {
this.filename = filename;
this.called.set(false);
}
@Override public void skip() throws IOException {
checkCall();
actions.put(filename, new EntryAction(ActionType.SKIP));
}
@Override public void copy(Date date) throws IOException {
checkCall();
actions.put(filename, new EntryAction(ActionType.COPY, date, null, null, null));
}
@Override public void rename(String newName, Date date) throws IOException {
checkCall();
actions.put(filename, new EntryAction(ActionType.RENAME, date, newName, null, null));
}
@Override public void customMerge(Date date, CustomMergeStrategy strategy) throws IOException {
checkCall();
actions.put(filename, new EntryAction(ActionType.MERGE, date, null, strategy,
new ByteArrayOutputStream()));
}
private void checkCall() {
checkState(called.compareAndSet(false, true), "The callback was already called once.");
}
}
/** Returns a {@link Deflater} for performing ZIP compression. */
private static Deflater getDeflater() {
return new Deflater(Deflater.DEFAULT_COMPRESSION, true);
}
/** Returns a {@link Inflater} for performing ZIP decompression. */
private static Inflater getInflater() {
return new Inflater(true);
}
/** Copies all data from the input stream to the output stream. */
private static long copyStream(InputStream from, OutputStream to) throws IOException {
byte[] buf = new byte[0x1000];
long total = 0;
int r;
while ((r = from.read(buf)) != -1) {
to.write(buf, 0, r);
total += r;
}
return total;
}
private final OutputMode mode;
private final ZipEntryFilter entryFilter;
private final FilterCallback callback;
private final ZipWriter out;
private final Map<String, ZipFileEntry> entries;
private final Map<String, EntryAction> actions;
/**
* Creates a {@link ZipCombiner} for combining ZIP files using the specified {@link OutputMode},
* {@link ZipEntryFilter}, and destination {@link OutputStream}.
*
* @param mode the compression preference for the output ZIP file
* @param entryFilter the filter to use when adding ZIP files to the combined output
* @param out the {@link OutputStream} for writing the combined ZIP file
*/
public ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out) {
this.mode = mode;
this.entryFilter = entryFilter;
this.callback = new FilterCallback();
this.out = new ZipWriter(new BufferedOutputStream(out), UTF_8);
this.entries = new HashMap<>();
this.actions = new HashMap<>();
}
/**
* Creates a {@link ZipCombiner} for combining ZIP files using the specified
* {@link ZipEntryFilter}, and destination {@link OutputStream}. Uses the DONT_CARE
* {@link OutputMode}.
*
* @param entryFilter the filter to use when adding ZIP files to the combined output
* @param out the {@link OutputStream} for writing the combined ZIP file
*/
public ZipCombiner(ZipEntryFilter entryFilter, OutputStream out) {
this(OutputMode.DONT_CARE, entryFilter, out);
}
/**
* Creates a {@link ZipCombiner} for combining ZIP files using the specified {@link OutputMode},
* and destination {@link OutputStream}. Uses a {@link CopyEntryFilter} as the
* {@link ZipEntryFilter}.
*
* @param mode the compression preference for the output ZIP file
* @param out the {@link OutputStream} for writing the combined ZIP file
*/
public ZipCombiner(OutputMode mode, OutputStream out) {
this(mode, new CopyEntryFilter(), out);
}
/**
* Creates a {@link ZipCombiner} for combining ZIP files using the specified destination
* {@link OutputStream}. Uses the DONT_CARE {@link OutputMode} and a {@link CopyEntryFilter} as
* the {@link ZipEntryFilter}.
*
* @param out the {@link OutputStream} for writing the combined ZIP file
*/
public ZipCombiner(OutputStream out) {
this(OutputMode.DONT_CARE, new CopyEntryFilter(), out);
}
/**
* Write all contents from the {@link InputStream} as a prefix file for the combined ZIP file.
*
* @param in the {@link InputStream} containing the prefix file data
* @throws IOException if there is an error writing the prefix file
*/
public void prependExecutable(InputStream in) throws IOException {
out.startPrefixFile();
copyStream(in, out);
out.endPrefixFile();
}
/**
* Adds a directory entry to the combined ZIP file using the specified filename and date.
*
* @param filename the name of the directory to create
* @param date the modified time to assign to the directory
* @throws IOException if there is an error writing the directory entry
*/
public void addDirectory(String filename, Date date) throws IOException {
addDirectory(filename, date, new ExtraData[0]);
}
/**
* Adds a directory entry to the combined ZIP file using the specified filename, date, and extra
* data.
*
* @param filename the name of the directory to create
* @param date the modified time to assign to the directory
* @param extra the extra field data to add to the directory entry
* @throws IOException if there is an error writing the directory entry
*/
public void addDirectory(String filename, Date date, ExtraData[] extra) throws IOException {
checkArgument(filename.endsWith("/"), "Directory names must end with a /");
checkState(!entries.containsKey(filename),
"Zip already contains a directory named %s", filename);
ZipFileEntry entry = new ZipFileEntry(filename);
entry.setMethod(Compression.STORED);
entry.setCrc(0);
entry.setSize(0);
entry.setCompressedSize(0);
entry.setTime(date != null ? date.getTime() : new Date().getTime());
entry.setExtra(new ExtraDataList(extra));
out.putNextEntry(entry);
out.closeEntry();
entries.put(filename, entry);
}
/**
* Adds a file with the specified name to the combined ZIP file.
*
* @param filename the name of the file to create
* @param in the {@link InputStream} containing the file data
* @throws IOException if there is an error writing the file entry
* @throws IllegalArgumentException if the combined ZIP file already contains a file of the same
* name.
*/
public void addFile(String filename, InputStream in) throws IOException {
addFile(filename, null, in);
}
/**
* Adds a file with the specified name and date to the combined ZIP file.
*
* @param filename the name of the file to create
* @param date the modified time to assign to the file
* @param in the {@link InputStream} containing the file data
* @throws IOException if there is an error writing the file entry
* @throws IllegalArgumentException if the combined ZIP file already contains a file of the same
* name.
*/
public void addFile(String filename, Date date, InputStream in) throws IOException {
ZipFileEntry entry = new ZipFileEntry(filename);
entry.setTime(date != null ? date.getTime() : new Date().getTime());
addFile(entry, in);
}
/**
* Adds a file with attributes specified by the {@link ZipFileEntry} to the combined ZIP file.
*
* @param entry the {@link ZipFileEntry} containing the entry meta-data
* @param in the {@link InputStream} containing the file data
* @throws IOException if there is an error writing the file entry
* @throws IllegalArgumentException if the combined ZIP file already contains a file of the same
* name.
*/
public void addFile(ZipFileEntry entry, InputStream in) throws IOException {
checkNotNull(entry, "Zip entry must not be null.");
checkNotNull(in, "Input stream must not be null.");
checkArgument(!entries.containsKey(entry.getName()), "Zip already contains a file named '%s'.",
entry.getName());
ByteArrayOutputStream uncompressed = new ByteArrayOutputStream();
copyStream(in, uncompressed);
writeEntryFromBuffer(new ZipFileEntry(entry), uncompressed.toByteArray());
}
/**
* Adds the contents of a ZIP file to the combined ZIP file using the specified
* {@link ZipEntryFilter} to determine the appropriate action for each file.
*
* @param zipFile the ZIP file to add to the combined ZIP file
* @throws IOException if there is an error reading the ZIP file or writing entries to the
* combined ZIP file
*/
public void addZip(File zipFile) throws IOException {
try (ZipReader zip = new ZipReader(zipFile)) {
for (ZipFileEntry entry : zip.entries()) {
String filename = entry.getName();
EntryAction action = getAction(filename);
switch (action.getType()) {
case SKIP:
break;
case COPY:
case RENAME:
writeEntry(zip, entry, action);
break;
case MERGE:
entries.put(filename, null);
InputStream in = zip.getRawInputStream(entry);
if (entry.getMethod() == Compression.DEFLATED) {
in = new InflaterInputStream(in, getInflater());
}
action.getStrategy().merge(in, action.getMergeBuffer());
break;
}
}
}
}
/** Returns the action to take for a file of the given filename. */
private EntryAction getAction(String filename) throws IOException {
// If this filename has not been encountered before (no entry for filename) or this filename
// has been renamed (RENAME entry for filename), the desired action should be recomputed.
if (!actions.containsKey(filename) || actions.get(filename).getType() == ActionType.RENAME) {
callback.resetForFile(filename);
entryFilter.accept(filename, callback);
}
checkState(actions.containsKey(filename),
"Action for file '%s' should have been set by ZipEntryFilter.", filename);
EntryAction action = actions.get(filename);
// Only copy if this is the first instance of filename.
if (action.getType() == ActionType.COPY && entries.containsKey(filename)) {
action = new EntryAction(ActionType.SKIP, action);
actions.put(filename, action);
}
// Only rename if there is not already an entry with filename or filename's action is SKIP.
if (action.getType() == ActionType.RENAME) {
if (actions.containsKey(action.getNewName())
&& actions.get(action.getNewName()).getType() == ActionType.SKIP) {
action = new EntryAction(ActionType.SKIP, action);
}
if (entries.containsKey(action.getNewName())) {
action = new EntryAction(ActionType.SKIP, action);
}
}
return action;
}
/** Writes an entry with the given name, date and external file attributes from the buffer. */
private void writeEntryFromBuffer(ZipFileEntry entry, byte[] uncompressed) throws IOException {
CRC32 crc = new CRC32();
crc.update(uncompressed);
entry.setCrc(crc.getValue());
entry.setSize(uncompressed.length);
if (mode == OutputMode.FORCE_STORED) {
entry.setMethod(Compression.STORED);
entry.setCompressedSize(uncompressed.length);
writeEntry(entry, new ByteArrayInputStream(uncompressed));
} else {
ByteArrayOutputStream compressed = new ByteArrayOutputStream();
copyStream(new DeflaterInputStream(new ByteArrayInputStream(uncompressed), getDeflater()),
compressed);
entry.setMethod(Compression.DEFLATED);
entry.setCompressedSize(compressed.size());
writeEntry(entry, new ByteArrayInputStream(compressed.toByteArray()));
}
}
/**
* Writes an entry from the specified source {@link ZipReader} and {@link ZipFileEntry} using the
* specified {@link EntryAction}.
*
* <p>Writes the output entry from the input entry performing inflation or deflation as needed
* and applies any values from the {@link EntryAction} as needed.
*/
private void writeEntry(ZipReader zip, ZipFileEntry entry, EntryAction action)
throws IOException {
checkArgument(action.getType() != ActionType.SKIP,
"Cannot write a zip entry whose action is of type SKIP.");
ZipFileEntry outEntry = new ZipFileEntry(entry);
if (action.getType() == ActionType.RENAME) {
checkNotNull(action.getNewName(),
"ZipEntryFilter actions of type RENAME must not have a null filename.");
outEntry.setName(action.getNewName());
}
if (action.getDate() != null) {
outEntry.setTime(action.getDate().getTime());
}
InputStream data;
if (mode == OutputMode.FORCE_DEFLATE && entry.getMethod() != Compression.DEFLATED) {
// The output mode is deflate, but the entry compression is not. Create a deflater stream
// from the raw file data and deflate to a temporary byte array to determine the deflated
// size. Then use this byte array as the input stream for writing the entry.
ByteArrayOutputStream tmp = new ByteArrayOutputStream();
copyStream(new DeflaterInputStream(zip.getRawInputStream(entry), getDeflater()), tmp);
data = new ByteArrayInputStream(tmp.toByteArray());
outEntry.setMethod(Compression.DEFLATED);
outEntry.setCompressedSize(tmp.size());
} else if (mode == OutputMode.FORCE_STORED && entry.getMethod() != Compression.STORED) {
// The output mode is stored, but the entry compression is not; create an inflater stream
// from the raw file data.
data = new InflaterInputStream(zip.getRawInputStream(entry), getInflater());
outEntry.setMethod(Compression.STORED);
outEntry.setCompressedSize(entry.getSize());
} else {
// Entry compression agrees with output mode; use the raw file data as is.
data = zip.getRawInputStream(entry);
}
writeEntry(outEntry, data);
}
/**
* Writes the specified {@link ZipFileEntry} using the data from the given {@link InputStream}.
*/
private void writeEntry(ZipFileEntry entry, InputStream data) throws IOException {
out.putNextEntry(entry);
copyStream(data, out);
out.closeEntry();
entries.put(entry.getName(), entry);
}
/**
* Returns true if the combined ZIP file already contains a file of the specified file name.
*
* @param filename the filename of the file whose presence in the combined ZIP file is to be
* tested
* @return true if the combined ZIP file contains the specified file
*/
public boolean containsFile(String filename) {
// TODO(apell): may be slightly different behavior because v1 returns true on skipped names.
return entries.containsKey(filename);
}
/**
* Writes any remaining output data to the output stream and also creates the merged entries by
* calling the {@link CustomMergeStrategy} implementations given back from the
* {@link ZipEntryFilter}.
*
* @throws IOException if the output stream or the filter throws an IOException
* @throws IllegalStateException if this method was already called earlier
*/
public void finish() throws IOException {
for (Entry<String, EntryAction> entry : actions.entrySet()) {
String filename = entry.getKey();
EntryAction action = entry.getValue();
if (action.getType() == ActionType.MERGE) {
ByteArrayOutputStream uncompressed = action.getMergeBuffer();
action.getStrategy().finish(uncompressed);
ZipFileEntry e = new ZipFileEntry(filename);
e.setTime(action.getDate() != null ? action.getDate().getTime() : new Date().getTime());
writeEntryFromBuffer(e, uncompressed.toByteArray());
}
}
out.finish();
}
/**
* Writes any remaining output data to the output stream and closes it.
*
* @throws IOException if the output stream or the filter throws an IOException
*/
@Override public void close() throws IOException {
finish();
out.close();
}
/** Ensures the truth of an expression involving one or more parameters to the calling method. */
private static void checkArgument(boolean expression,
@Nullable String errorMessageTemplate,
@Nullable Object... errorMessageArgs) {
if (!expression) {
throw new IllegalArgumentException(String.format(errorMessageTemplate, errorMessageArgs));
}
}
/** Ensures that an object reference passed as a parameter to the calling method is not null. */
public static <T> T checkNotNull(T reference,
@Nullable String errorMessageTemplate,
@Nullable Object... errorMessageArgs) {
if (reference == null) {
// If either of these parameters is null, the right thing happens anyway
throw new NullPointerException(String.format(errorMessageTemplate, errorMessageArgs));
}
return reference;
}
/** Ensures the truth of an expression involving state. */
private static void checkState(boolean expression,
@Nullable String errorMessageTemplate,
@Nullable Object... errorMessageArgs) {
if (!expression) {
throw new IllegalStateException(String.format(errorMessageTemplate, errorMessageArgs));
}
}
}