/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.cassandra.io.util; import java.io.BufferedWriter; import java.io.Closeable; import java.io.DataOutput; import java.io.File; import java.io.IOError; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.zip.CRC32; import java.util.zip.CheckedInputStream; import java.util.zip.Checksum; import com.google.common.base.Charsets; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.utils.Throwables; public class DataIntegrityMetadata { public static ChecksumValidator checksumValidator(Descriptor desc) throws IOException { return new ChecksumValidator(desc); } public static class ChecksumValidator implements Closeable { private final Checksum checksum; private final RandomAccessReader reader; public final int chunkSize; private final String dataFilename; public ChecksumValidator(Descriptor descriptor) throws IOException { this(descriptor.version.uncompressedChecksumType().newInstance(), RandomAccessReader.open(new File(descriptor.filenameFor(Component.CRC))), descriptor.filenameFor(Component.DATA)); } public ChecksumValidator(Checksum checksum, RandomAccessReader reader, String dataFilename) throws IOException { this.checksum = checksum; this.reader = reader; this.dataFilename = dataFilename; chunkSize = reader.readInt(); } public void seek(long offset) { long start = chunkStart(offset); reader.seek(((start / chunkSize) * 4L) + 4); // 8 byte checksum per chunk + 4 byte header/chunkLength } public long chunkStart(long offset) { long startChunk = offset / chunkSize; return startChunk * chunkSize; } public void validate(byte[] bytes, int start, int end) throws IOException { checksum.update(bytes, start, end); int current = (int) checksum.getValue(); checksum.reset(); int actual = reader.readInt(); if (current != actual) throw new IOException("Corrupted File : " + dataFilename); } public void close() { reader.close(); } } public static FileDigestValidator fileDigestValidator(Descriptor desc) throws IOException { return new FileDigestValidator(desc); } public static class FileDigestValidator implements Closeable { private final Checksum checksum; private final RandomAccessReader digestReader; private final RandomAccessReader dataReader; private final Descriptor descriptor; private long storedDigestValue; public FileDigestValidator(Descriptor descriptor) throws IOException { this.descriptor = descriptor; checksum = descriptor.version.uncompressedChecksumType().newInstance(); digestReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.digestFor(descriptor.version.uncompressedChecksumType())))); dataReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DATA))); try { storedDigestValue = Long.parseLong(digestReader.readLine()); } catch (Exception e) { close(); // Attempting to create a FileDigestValidator without a DIGEST file will fail throw new IOException("Corrupted SSTable : " + descriptor.filenameFor(Component.DATA)); } } // Validate the entire file public void validate() throws IOException { CheckedInputStream checkedInputStream = new CheckedInputStream(dataReader, checksum); byte[] chunk = new byte[64 * 1024]; while( checkedInputStream.read(chunk) > 0 ) { } long calculatedDigestValue = checkedInputStream.getChecksum().getValue(); if (storedDigestValue != calculatedDigestValue) { throw new IOException("Corrupted SSTable : " + descriptor.filenameFor(Component.DATA)); } } public void close() { Throwables.perform(digestReader::close, dataReader::close); } } public static class ChecksumWriter { private final CRC32 incrementalChecksum = new CRC32(); private final DataOutput incrementalOut; private final CRC32 fullChecksum = new CRC32(); public ChecksumWriter(DataOutput incrementalOut) { this.incrementalOut = incrementalOut; } public void writeChunkSize(int length) { try { incrementalOut.writeInt(length); } catch (IOException e) { throw new IOError(e); } } // checksumIncrementalResult indicates if the checksum we compute for this buffer should itself be // included in the full checksum, translating to if the partial checksum is serialized along with the // data it checksums (in which case the file checksum as calculated by external tools would mismatch if // we did not include it), or independently. // CompressedSequentialWriters serialize the partial checksums inline with the compressed data chunks they // corroborate, whereas ChecksummedSequentialWriters serialize them to a different file. public void appendDirect(ByteBuffer bb, boolean checksumIncrementalResult) { try { ByteBuffer toAppend = bb.duplicate(); toAppend.mark(); incrementalChecksum.update(toAppend); toAppend.reset(); int incrementalChecksumValue = (int) incrementalChecksum.getValue(); incrementalOut.writeInt(incrementalChecksumValue); fullChecksum.update(toAppend); if (checksumIncrementalResult) { ByteBuffer byteBuffer = ByteBuffer.allocate(4); byteBuffer.putInt(incrementalChecksumValue); fullChecksum.update(byteBuffer.array(), 0, byteBuffer.array().length); } incrementalChecksum.reset(); } catch (IOException e) { throw new IOError(e); } } public void writeFullChecksum(Descriptor descriptor) { if (descriptor.digestComponent == null) throw new NullPointerException("Null digest component for " + descriptor.ksname + '.' + descriptor.cfname + " file " + descriptor.baseFilename()); File outFile = new File(descriptor.filenameFor(descriptor.digestComponent)); try (BufferedWriter out =Files.newBufferedWriter(outFile.toPath(), Charsets.UTF_8)) { out.write(String.valueOf(fullChecksum.getValue())); } catch (IOException e) { throw new FSWriteError(e, outFile); } } } }