/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.core.util; import com.linkedin.pinot.common.Utils; import com.linkedin.pinot.core.segment.creator.impl.V1Constants; import java.io.BufferedInputStream; import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.zip.Adler32; import java.util.zip.CheckedInputStream; import java.util.zip.Checksum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Dec 4, 2014 */ public class CrcUtils { private static final Logger LOGGER = LoggerFactory.getLogger(CrcUtils.class); private static final int BUFFER_SIZE = 65536; private final List<File> filesToProcess; private CrcUtils(List<File> files) { filesToProcess = files; } public static CrcUtils forFile(File file) { final List<File> files = new ArrayList<File>(); files.add(file); return new CrcUtils(files); } public static CrcUtils forAllFilesInFolder(File dir) { final File[] allFiles = dir.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { if (pathname.getName().equals(V1Constants.SEGMENT_CREATION_META)) { return false; } return true; } }); Arrays.sort(allFiles); final List<File> files = new ArrayList<File>(); for (final File f : allFiles) { files.add(f); } return new CrcUtils(files); } public long computeCrc() { CheckedInputStream cis = null; InputStream is = null; final Checksum checksum = new Adler32(); final byte[] tempBuf = new byte[BUFFER_SIZE]; for (final File file : filesToProcess) { try { is = new BufferedInputStream(new FileInputStream(file)); cis = new CheckedInputStream(is, checksum); while (cis.read(tempBuf) >= 0) { } cis.close(); is.close(); } catch (final Exception e) { LOGGER.error("Caught exception while computing CRC", e); Utils.rethrowException(e); throw new AssertionError("Should not reach this"); } } return checksum.getValue(); } public String computeMD5() throws NoSuchAlgorithmException, IOException { final MessageDigest digest = MessageDigest.getInstance("md5"); for (final File file : filesToProcess) { try { final FileInputStream f = new FileInputStream(file); final byte[] buffer = new byte[8192]; int len = 0; while (-1 != (len = f.read(buffer))) { digest.update(buffer, 0, len); } f.close(); } catch (final Exception e) { LOGGER.error("Caught exception while computing MD5", e); Utils.rethrowException(e); throw new AssertionError("Should not reach this"); } } return toHexaDecimal(digest.digest()); } public static String toHexaDecimal(byte[] bytesToConvert) { final char[] hexCharactersAsArray = "0123456789ABCDEF".toCharArray(); final char[] convertedHexCharsArray = new char[bytesToConvert.length * 2]; for (int j = 0; j < bytesToConvert.length; j++) { final int v = bytesToConvert[j] & 0xFF; convertedHexCharsArray[j * 2] = hexCharactersAsArray[v >>> 4]; convertedHexCharsArray[j * 2 + 1] = hexCharactersAsArray[v & 0x0F]; } return new String(convertedHexCharsArray); } }