/*
* Copyright 2014 WANdisco
*
* WANdisco licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package c5db.log;
import c5db.util.CrcInputStream;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.primitives.Ints;
import io.protostuff.LinkBuffer;
import io.protostuff.LowCopyProtobufOutput;
import io.protostuff.ProtobufIOUtil;
import io.protostuff.Schema;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
import java.util.zip.Adler32;
import static com.google.common.math.IntMath.checkedAdd;
/**
* Contains methods used for encoding and decoding log entries
*/
public class EntryEncodingUtil {
/**
* Exception indicating that a CRC has been read which does not match up with
* the CRC computed from the associated data.
*/
public static class CrcError extends RuntimeException {
public CrcError(String s) {
super(s);
}
}
/**
* Serialize a protostuff message object, prefixed with message length, and suffixed with a 4-byte CRC.
*
* @param schema Protostuff message schema
* @param message Object to serialize
* @param <T> Message type
* @return A list of ByteBuffers containing a varInt length, followed by the message, followed by a 4-byte CRC.
*/
public static <T> List<ByteBuffer> encodeWithLengthAndCrc(Schema<T> schema, T message) {
final LinkBuffer messageBuf = new LinkBuffer();
final LowCopyProtobufOutput lcpo = new LowCopyProtobufOutput(messageBuf);
try {
schema.writeTo(lcpo, message);
final int length = Ints.checkedCast(lcpo.buffer.size());
final LinkBuffer lengthBuf = new LinkBuffer().writeVarInt32(length);
return appendCrcToBufferList(
Lists.newArrayList(
Iterables.concat(lengthBuf.finish(), messageBuf.finish()))
);
} catch (IOException e) {
// This method performs no IO, so it should not actually be possible for an IOException to be thrown.
// But just in case...
throw new RuntimeException(e);
}
}
/**
* Decode a message from the passed input stream, and compute and verify its CRC. This method reads
* data written by the method {@link EntryEncodingUtil#encodeWithLengthAndCrc}.
*
* @param inputStream Input stream, opened for reading and positioned just before the length-prepended header
* @return The deserialized, constructed, validated message
* @throws IOException if a problem is encountered while reading or parsing
* @throws EntryEncodingUtil.CrcError if the recorded CRC of the message does not match its computed CRC.
*/
public static <T> T decodeAndCheckCrc(InputStream inputStream, Schema<T> schema)
throws IOException, CrcError {
// TODO this should check the length first and compare it with a passed-in maximum length
final T message = schema.newMessage();
final CrcInputStream crcStream = new CrcInputStream(inputStream, new Adler32());
ProtobufIOUtil.mergeDelimitedFrom(crcStream, message, schema);
final long computedCrc = crcStream.getValue();
final long diskCrc = readCrc(inputStream);
if (diskCrc != computedCrc) {
throw new CrcError("CRC mismatch on deserialized message " + message.toString());
}
return message;
}
/**
* Given a list of ByteBuffers, compute the combined CRC and then append it to the list as one or more
* additional ByteBuffers. Return the entire resulting collection as a new list, including the original
* ByteBuffers.
*
* @param content non-null list of ByteBuffers; no mutation will be performed on them.
* @return New list of ByteBuffers, with the CRC appended to the original ByteBuffers
*/
public static List<ByteBuffer> appendCrcToBufferList(List<ByteBuffer> content) throws IOException {
assert content != null;
final Adler32 crc = new Adler32();
content.forEach((ByteBuffer buffer) -> crc.update(buffer.duplicate()));
final LinkBuffer crcBuf = new LinkBuffer(8);
putCrc(crcBuf, crc.getValue());
return Lists.newArrayList(Iterables.concat(content, crcBuf.finish()));
}
/**
* Write a passed CRC to the passed buffer. The CRC is a 4-byte unsigned integer stored in a long; write it
* as (fixed length) 4 bytes.
*
* @param writeTo Buffer to write to; exactly 4 bytes will be written.
* @param crc CRC to write; caller guarantees that the code is within the range:
* 0 <= CRC < 2^32
*/
private static void putCrc(final LinkBuffer writeTo, final long crc) throws IOException {
// To store the CRC in an int, we need to subtract to convert it from unsigned to signed.
final long shiftedCrc = crc + Integer.MIN_VALUE;
writeTo.writeInt32(Ints.checkedCast(shiftedCrc));
}
private static long readCrc(InputStream inputStream) throws IOException {
int shiftedCrc = (new DataInputStream(inputStream)).readInt();
return ((long) shiftedCrc) - Integer.MIN_VALUE;
}
/**
* Read a specified number of bytes from the input stream (the "content"), then read one or more CRC codes and
* check the validity of the data.
*
* @param inputStream Input stream, opened for reading and positioned just before the content
* @param contentLength Length of data to read from inputStream, not including any trailing CRCs
* @return The read content, as a ByteBuffer.
* @throws IOException
*/
public static ByteBuffer getAndCheckContent(InputStream inputStream, int contentLength)
throws IOException, CrcError {
// TODO probably not the correct way to do this... should use IOUtils?
final CrcInputStream crcStream = new CrcInputStream(inputStream, new Adler32());
final byte[] content = new byte[contentLength];
final int len = crcStream.read(content);
if (len < contentLength) {
// Data wasn't available that we expected to be
throw new IllegalStateException("Reading a log entry's contents returned fewer than expected bytes");
}
final long computedCrc = crcStream.getValue();
final long diskCrc = readCrc(inputStream);
if (diskCrc != computedCrc) {
throw new CrcError("CRC mismatch on log entry contents");
}
return ByteBuffer.wrap(content);
}
public static void skip(InputStream inputStream, int numBytes) throws IOException {
long actuallySkipped = inputStream.skip(numBytes);
if (actuallySkipped < numBytes) {
throw new IOException("Unable to skip requested number of bytes");
}
}
/**
* Add up the lengths of the content of each buffer in the passed list, and return the sum of the lengths.
*
* @param buffers List of buffers; this method will not mutate them.
* @return The sum of the remaining() bytes in each buffer.
*/
public static int sumRemaining(Collection<ByteBuffer> buffers) {
int length = 0;
if (buffers != null) {
for (ByteBuffer b : buffers) {
length = checkedAdd(length, b.remaining());
}
}
return length;
}
}