/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package parquet.hadoop; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.CompressionInputStream; import org.apache.hadoop.io.compress.CompressionOutputStream; import org.junit.Test; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import org.xerial.snappy.Snappy; import parquet.hadoop.codec.SnappyCodec; import parquet.hadoop.codec.SnappyCompressor; import parquet.hadoop.codec.SnappyDecompressor; public class TestSnappyCodec { @Test public void TestSnappy() throws IOException { // Reuse the snappy objects between test cases SnappyCompressor compressor = new SnappyCompressor(); SnappyDecompressor decompressor = new SnappyDecompressor(); TestSnappy(compressor, decompressor, ""); TestSnappy(compressor, decompressor, "FooBar"); TestSnappy(compressor, decompressor, "FooBar1", "FooBar2"); TestSnappy(compressor, decompressor, "FooBar"); TestSnappy(compressor, decompressor, "a", "blahblahblah", "abcdef"); TestSnappy(compressor, decompressor, ""); TestSnappy(compressor, decompressor, "FooBar"); } @Test public void TestSnappyStream() throws IOException { SnappyCodec codec = new SnappyCodec(); codec.setConf(new Configuration()); int blockSize = 1024; int inputSize = blockSize * 1024; byte[] input = new byte[inputSize]; for (int i = 0; i < inputSize; ++i) { input[i] = (byte) i; } ByteArrayOutputStream compressedStream = new ByteArrayOutputStream(); CompressionOutputStream compressor = codec.createOutputStream(compressedStream); int bytesCompressed = 0; while (bytesCompressed < inputSize) { int len = Math.min(inputSize - bytesCompressed, blockSize); compressor.write(input, bytesCompressed, len); bytesCompressed += len; } compressor.finish(); byte[] rawCompressed = Snappy.compress(input); byte[] codecCompressed = compressedStream.toByteArray(); // Validate that the result from the codec is the same as if we compressed the // buffer directly. assertArrayEquals(rawCompressed, codecCompressed); ByteArrayInputStream inputStream = new ByteArrayInputStream(codecCompressed); CompressionInputStream decompressor = codec.createInputStream(inputStream); byte[] codecDecompressed = new byte[inputSize]; int bytesDecompressed = 0; int numBytes; while ((numBytes = decompressor.read(codecDecompressed, bytesDecompressed, blockSize)) != 0) { bytesDecompressed += numBytes; if (bytesDecompressed == inputSize) break; } byte[] rawDecompressed = Snappy.uncompress(rawCompressed); assertArrayEquals(input, rawDecompressed); assertArrayEquals(input, codecDecompressed); } private void TestSnappy(SnappyCompressor compressor, SnappyDecompressor decompressor, String... strings) throws IOException { compressor.reset(); decompressor.reset(); int uncompressedSize = 0; for (String s : strings) { uncompressedSize += s.length(); } byte[] uncompressedData = new byte[uncompressedSize]; int len = 0; for (String s : strings) { byte[] tmp = s.getBytes(); System.arraycopy(tmp, 0, uncompressedData, len, s.length()); len += s.length(); } assert (compressor.needsInput()); compressor.setInput(uncompressedData, 0, len); assert (compressor.needsInput()); compressor.finish(); assert (!compressor.needsInput()); assert (!compressor.finished() || uncompressedSize == 0); byte[] compressedData = new byte[1000]; int compressedSize = compressor.compress(compressedData, 0, 1000); assert (compressor.finished()); assert (!decompressor.finished()); assert (decompressor.needsInput()); decompressor.setInput(compressedData, 0, compressedSize); assert (!decompressor.finished()); byte[] decompressedData = new byte[uncompressedSize]; int decompressedSize = decompressor.decompress(decompressedData, 0, uncompressedSize); assert (decompressor.finished()); assertEquals(uncompressedSize, decompressedSize); assertArrayEquals(uncompressedData, decompressedData); } }