// Copyright 2015 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.singlejar;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.CRC32;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;
/**
* A helper class to validate zip files and provide reasonable diagnostics (better than what zip
* does). We might want to make this into a fully-fledged binary some day.
*/
final class ZipTester {
// The following constants are ZIP-specific.
private static final int LOCAL_FILE_HEADER_MARKER = 0x04034b50;
private static final int DATA_DESCRIPTOR_MARKER = 0x08074b50;
private static final int CENTRAL_DIRECTORY_MARKER = 0x02014b50;
private static final int END_OF_CENTRAL_DIRECTORY_MARKER = 0x06054b50;
private static final int FILE_HEADER_BUFFER_SIZE = 26; // without marker
private static final int DATA_DESCRIPTOR_BUFFER_SIZE = 12; // without marker
private static final int DIRECTORY_ENTRY_BUFFER_SIZE = 42; // without marker
private static final int END_OF_CENTRAL_DIRECTORY_BUFFER_SIZE = 18; // without marker
// Set if the size, compressed size and CRC are set to zero, and present in
// the data descriptor after the data.
private static final int SIZE_MASKED_FLAG = 1 << 3;
private static final int STORED_METHOD = 0;
private static final int DEFLATE_METHOD = 8;
private static final int VERSION_STORED = 10; // Version 1.0
private static final int VERSION_DEFLATE = 20; // Version 2.0
private static class Entry {
private final long pos;
private final String name;
private final int flags;
private final int method;
private final int dosTime;
Entry(long pos, String name, int flags, int method, int dosTime) {
this.pos = pos;
this.name = name;
this.flags = flags;
this.method = method;
this.dosTime = dosTime;
}
}
private final InputStream in;
private final byte[] buffer = new byte[1024];
private int bufferLength;
private int bufferOffset;
private long pos;
private List<Entry> entries = new ArrayList<Entry>();
public ZipTester(InputStream in) {
this.in = in;
}
public ZipTester(byte[] data) {
this(new ByteArrayInputStream(data));
}
private void warn(String msg) {
System.err.println("WARNING: " + msg);
}
private void readMoreData(String action) throws IOException {
if ((bufferLength > 0) && (bufferOffset > 0)) {
System.arraycopy(buffer, bufferOffset, buffer, 0, bufferLength);
}
if (bufferLength >= buffer.length) {
// The buffer size is specifically chosen to avoid this situation.
throw new AssertionError("Internal error: buffer overrun.");
}
bufferOffset = 0;
int bytesRead = in.read(buffer, bufferLength, buffer.length - bufferLength);
if (bytesRead <= 0) {
throw new IOException("Unexpected end of file, while " + action);
}
bufferLength += bytesRead;
}
private int readByte(String action) throws IOException {
if (bufferLength == 0) {
readMoreData(action);
}
byte result = buffer[bufferOffset];
bufferOffset++; bufferLength--;
pos++;
return result & 0xff;
}
private long getUnsignedInt(String action) throws IOException {
int a = readByte(action);
int b = readByte(action);
int c = readByte(action);
int d = readByte(action);
return ((d << 24) | (c << 16) | (b << 8) | a) & 0xffffffffL;
}
private void readFully(byte[] buffer, String action) throws IOException {
for (int i = 0; i < buffer.length; i++) {
buffer[i] = (byte) readByte(action);
}
}
private void skip(long length, String action) throws IOException {
for (long i = 0; i < length; i++) {
readByte(action);
}
}
private int getUnsignedShort(byte[] source, int offset) {
int a = source[offset + 0] & 0xff;
int b = source[offset + 1] & 0xff;
return (b << 8) | a;
}
private long getUnsignedInt(byte[] source, int offset) {
int a = source[offset + 0] & 0xff;
int b = source[offset + 1] & 0xff;
int c = source[offset + 2] & 0xff;
int d = source[offset + 3] & 0xff;
return ((d << 24) | (c << 16) | (b << 8) | a) & 0xffffffffL;
}
private class DeflateInputStream extends InputStream {
private final byte[] singleByteBuffer = new byte[1];
private int consumedBytes;
private final Inflater inflater = new Inflater(true);
private long totalBytesRead;
private int inflateData(byte[] dest, int off, int len)
throws IOException {
consumedBytes = 0;
int bytesProduced = 0;
int bytesConsumed = 0;
while ((bytesProduced == 0) && !inflater.finished()) {
inflater.setInput(buffer, bufferOffset + bytesConsumed, bufferLength - bytesConsumed);
int remainingBefore = inflater.getRemaining();
try {
bytesProduced = inflater.inflate(dest, off, len);
} catch (DataFormatException e) {
throw new IOException("Invalid deflate stream in ZIP file.", e);
}
bytesConsumed += remainingBefore - inflater.getRemaining();
consumedBytes = bytesConsumed;
if (bytesProduced == 0) {
if (inflater.needsDictionary()) {
// The DEFLATE algorithm as used in the ZIP file format does not
// require an additional dictionary.
throw new AssertionError("Inflater unexpectedly requires a dictionary.");
} else if (inflater.needsInput()) {
readMoreData("need more data for deflate");
} else if (inflater.finished()) {
return 0;
} else {
// According to the Inflater specification, this cannot happen.
throw new AssertionError("Inflater unexpectedly produced no output.");
}
}
}
return bytesProduced;
}
@Override
public int read(byte b[], int off, int len) throws IOException {
if (inflater.finished()) {
return -1;
}
int length = inflateData(b, off, len);
totalBytesRead += consumedBytes;
bufferLength -= consumedBytes;
bufferOffset += consumedBytes;
pos += consumedBytes;
return length == 0 ? -1 : length;
}
@Override
public int read() throws IOException {
int bytesRead = read(singleByteBuffer, 0, 1);
return (bytesRead == -1) ? -1 : (singleByteBuffer[0] & 0xff);
}
}
private void readEntry() throws IOException {
long entrypos = pos - 4;
String entryDesc = "file entry at " + Long.toHexString(entrypos);
byte[] entryBuffer = new byte[FILE_HEADER_BUFFER_SIZE];
readFully(entryBuffer, "reading file header");
int versionToExtract = getUnsignedShort(entryBuffer, 0);
int flags = getUnsignedShort(entryBuffer, 2);
int method = getUnsignedShort(entryBuffer, 4);
int dosTime = (int) getUnsignedInt(entryBuffer, 6);
int crc32 = (int) getUnsignedInt(entryBuffer, 10);
long compressedSize = getUnsignedInt(entryBuffer, 14);
long uncompressedSize = getUnsignedInt(entryBuffer, 18);
int filenameLength = getUnsignedShort(entryBuffer, 22);
int extraLength = getUnsignedShort(entryBuffer, 24);
byte[] filename = new byte[filenameLength];
readFully(filename, "reading file name");
skip(extraLength, "skipping extra data");
String name = new String(filename, "UTF-8");
for (int i = 0; i < filename.length; i++) {
if ((filename[i] < ' ') || (filename[i] > 127)) {
warn(entryDesc + ": file name has unexpected non-ascii characters");
}
}
entryDesc = "file entry '" + name + "' at " + Long.toHexString(entrypos);
if ((method != STORED_METHOD) && (method != DEFLATE_METHOD)) {
throw new IOException(entryDesc + ": unknown method " + method);
}
if ((flags != 0) && (flags != SIZE_MASKED_FLAG)) {
throw new IOException(entryDesc + ": unknown flags " + flags);
}
if ((method == STORED_METHOD) && (versionToExtract != VERSION_STORED)) {
warn(entryDesc + ": unexpected version to extract for stored entry " + versionToExtract);
}
if ((method == DEFLATE_METHOD) && (versionToExtract != VERSION_DEFLATE)) {
// warn(entryDesc + ": unexpected version to extract for deflated entry " + versionToExtract);
}
if (method == STORED_METHOD) {
if (compressedSize != uncompressedSize) {
throw new IOException(entryDesc + ": stored entries should have identical compressed and "
+ "uncompressed sizes");
}
skip(compressedSize, entryDesc + "skipping data");
} else {
// No OS resources are actually allocated.
@SuppressWarnings("resource") DeflateInputStream deflater = new DeflateInputStream();
long generatedBytes = 0;
byte[] deflated = new byte[1024];
int readBytes;
CRC32 crc = new CRC32();
while ((readBytes = deflater.read(deflated)) > 0) {
crc.update(deflated, 0, readBytes);
generatedBytes += readBytes;
}
int actualCrc32 = (int) crc.getValue();
long consumedBytes = deflater.totalBytesRead;
if (flags == SIZE_MASKED_FLAG) {
long id = getUnsignedInt("reading footer marker");
if (id != DATA_DESCRIPTOR_MARKER) {
throw new IOException(entryDesc + ": expected footer at " + Long.toHexString(pos - 4)
+ ", but found " + Long.toHexString(id));
}
byte[] footer = new byte[DATA_DESCRIPTOR_BUFFER_SIZE];
readFully(footer, "reading footer");
crc32 = (int) getUnsignedInt(footer, 0);
compressedSize = getUnsignedInt(footer, 4);
uncompressedSize = getUnsignedInt(footer, 8);
}
if (consumedBytes != compressedSize) {
throw new IOException(entryDesc + ": amount of compressed data does not match value "
+ "specified in the zip (specified: " + compressedSize + ", actual: " + consumedBytes
+ ")");
}
if (generatedBytes != uncompressedSize) {
throw new IOException(entryDesc + ": amount of uncompressed data does not match value "
+ "specified in the zip (specified: " + uncompressedSize + ", actual: "
+ generatedBytes + ")");
}
if (crc32 != actualCrc32) {
throw new IOException(entryDesc + ": specified crc checksum does not match actual check "
+ "sum");
}
}
entries.add(new Entry(entrypos, name, flags, method, dosTime));
}
@SuppressWarnings("unused") // A couple of unused local variables.
private void validateCentralDirectoryEntry(Entry entry) throws IOException {
long entrypos = pos - 4;
String entryDesc = "file directory entry '" + entry.name + "' at " + Long.toHexString(entrypos);
byte[] entryBuffer = new byte[DIRECTORY_ENTRY_BUFFER_SIZE];
readFully(entryBuffer, "reading central directory entry");
int versionMadeBy = getUnsignedShort(entryBuffer, 0);
int versionToExtract = getUnsignedShort(entryBuffer, 2);
int flags = getUnsignedShort(entryBuffer, 4);
int method = getUnsignedShort(entryBuffer, 6);
int dosTime = (int) getUnsignedInt(entryBuffer, 8);
int crc32 = (int) getUnsignedInt(entryBuffer, 12);
long compressedSize = getUnsignedInt(entryBuffer, 16);
long uncompressedSize = getUnsignedInt(entryBuffer, 20);
int filenameLength = getUnsignedShort(entryBuffer, 24);
int extraLength = getUnsignedShort(entryBuffer, 26);
int commentLength = getUnsignedShort(entryBuffer, 28);
int diskNumberStart = getUnsignedShort(entryBuffer, 30);
int internalAttributes = getUnsignedShort(entryBuffer, 32);
int externalAttributes = (int) getUnsignedInt(entryBuffer, 34);
long offset = getUnsignedInt(entryBuffer, 38);
byte[] filename = new byte[filenameLength];
readFully(filename, "reading file name");
skip(extraLength, "skipping extra data");
String name = new String(filename, "UTF-8");
if (!name.equals(entry.name)) {
throw new IOException(entryDesc + ": file name in central directory does not match original "
+ "name");
}
if (offset != entry.pos) {
throw new IOException(entryDesc);
}
if (flags != entry.flags) {
throw new IOException(entryDesc);
}
if (method != entry.method) {
throw new IOException(entryDesc);
}
if (dosTime != entry.dosTime) {
throw new IOException(entryDesc);
}
}
private void validateCentralDirectory() throws IOException {
boolean first = true;
for (Entry entry : entries) {
if (first) {
first = false;
} else {
long id = getUnsignedInt("reading marker");
if (id != CENTRAL_DIRECTORY_MARKER) {
throw new IOException();
}
}
validateCentralDirectoryEntry(entry);
}
}
@SuppressWarnings("unused") // A couple of unused local variables.
private void validateEndOfCentralDirectory() throws IOException {
long id = getUnsignedInt("expecting end of central directory");
byte[] entryBuffer = new byte[END_OF_CENTRAL_DIRECTORY_BUFFER_SIZE];
readFully(entryBuffer, "reading end of central directory");
int diskNumber = getUnsignedShort(entryBuffer, 0);
int startDiskNumber = getUnsignedShort(entryBuffer, 2);
int numEntries = getUnsignedShort(entryBuffer, 4);
int numTotalEntries = getUnsignedShort(entryBuffer, 6);
long centralDirectorySize = getUnsignedInt(entryBuffer, 8);
long centralDirectoryOffset = getUnsignedInt(entryBuffer, 12);
int commentLength = getUnsignedShort(entryBuffer, 16);
if (diskNumber != 0) {
throw new IOException(String.format("diskNumber=%d", diskNumber));
}
if (startDiskNumber != 0) {
throw new IOException(String.format("startDiskNumber=%d", diskNumber));
}
if (numEntries != numTotalEntries) {
throw new IOException(String.format("numEntries=%d numTotalEntries=%d",
numEntries, numTotalEntries));
}
if (numEntries != (entries.size() % 0x10000)) {
throw new IOException("bad number of entries in central directory footer");
}
if (numTotalEntries != (entries.size() % 0x10000)) {
throw new IOException("bad number of entries in central directory footer");
}
if (commentLength != 0) {
throw new IOException("Zip file comment is unexpected");
}
if (id != END_OF_CENTRAL_DIRECTORY_MARKER) {
throw new IOException("Expected end of central directory marker");
}
}
public void validate() throws IOException {
while (true) {
long id = getUnsignedInt("reading marker");
if (id == LOCAL_FILE_HEADER_MARKER) {
readEntry();
} else if (id == CENTRAL_DIRECTORY_MARKER) {
validateCentralDirectory();
validateEndOfCentralDirectory();
return;
} else {
throw new IOException("unexpected result for marker: "
+ Long.toHexString(id) + " at position " + Long.toHexString(pos - 4));
}
}
}
}