/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.InjectionEventCore;
import org.apache.hadoop.util.InjectionEventI;
import org.apache.hadoop.util.InjectionHandler;
import static org.junit.Assert.*;
import org.junit.After;
import org.junit.Test;
public class TestBufferedByteInputOutput {
static final Log LOG = LogFactory.getLog(TestBufferedByteInputOutput.class
.getName());
private byte[] input;
private byte[] output;
private Random rand = new Random();
private void setUp(int inputSize) {
input = new byte[inputSize];
output = new byte[inputSize];
rand.nextBytes(input);
}
@After
public void tearDown() {
InjectionHandler.clear();
}
/**
* Test raw buffer writes and reads.
*/
@Test
public void testBuffer() throws Exception {
for (int i = 0; i < 100; i++) {
testBuffer(rand(1000), rand(1000), -1, true);
}
}
/**
* Test basic functionality
*/
@Test
public void testBufferBasic() throws Exception {
setUp(50);
final BufferedByteInputOutput buffer = new BufferedByteInputOutput(50);
// empty buffer
assertEquals(0, buffer.available());
assertEquals(0, buffer.totalRead());
assertEquals(0, buffer.totalWritten());
// write 20 bytes
buffer.write(input, 0, 20);
assertEquals(20, buffer.available());
assertEquals(0, buffer.totalRead());
assertEquals(20, buffer.totalWritten());
// read 10 bytes
buffer.read(output, 0, 10);
assertEquals(10, buffer.available());
assertEquals(10, buffer.totalRead());
assertEquals(20, buffer.totalWritten());
// no more writes should be accepted
// we still should be able to read 10 bytes
buffer.close();
try {
buffer.write(1);
fail("Should not accept writes");
} catch (Exception e) {
LOG.info("Expected exception");
}
// try to read 20 bytes
assertEquals(10, buffer.available());
assertEquals(10, buffer.read(output, 10, 20));
// next read should return -1
assertEquals(0, buffer.available());
assertEquals(-1, buffer.read(output, 20, 5));
assertEquals(-1, buffer.read());
}
/**
* Can be used for testing throughput on large buffer.
*/
@Test
public void testBufferThroughput() throws Exception {
LOG.info("Test buffer throughput");
long time = testBuffer(1024 * 1024, 10 * 1024, 1024, false);
LOG.info("Time taken: " + time);
}
/**
* Test pipeline where background thread reads from underlying stream.
*/
@Test
public void testInputStream() throws IOException {
for (int i = 0; i < 100; i++) {
testInputStream(rand(1000), rand(1000), rand(1000));
}
}
/**
* Test pipeline where background thread writes to underlying stream.
*/
@Test
public void testOutputStream() throws IOException {
// do close after completing writes
for (int i = 0; i < 100; i++) {
testOutputStream(rand(1000), rand(1000), rand(1000), true);
}
// do flush after completing writes
for (int i = 0; i < 100; i++) {
testOutputStream(rand(1000), rand(1000), rand(1000), false);
}
}
/**
* Test reading from closed buffer.
*/
@Test
public void testCloseInput() throws IOException {
LOG.info("Running test close input");
setUp(1000);
// input is of size 1000, so the ReadThread will attempt to write to
// the buffer, which will fail, but we should be able to read 100 bytes
ByteArrayInputStream is = new ByteArrayInputStream(input);
DataInputStream dis = BufferedByteInputStream.wrapInputStream(is, 100, 10);
// wait for the thread to read from is and
// write to the buffer
while(dis.available() < 100) {
sleep(10);
}
// no more writes to the internal buffer
dis.close();
try {
dis.read(); // read will call DataInputStream fill() which should fail
fail("Read should fail because we are closed");
} catch (Exception e) {
LOG.info("Expected exception " + e.getMessage());
}
dis.close(); // can call multiple close()
try {
dis.read(new byte[10], 0, 10);
fail("Read should fail because we are closed");
} catch (Exception e) {
LOG.info("Expected exception " + e.getMessage());
}
try {
dis.available();
fail("Available should fail because we are closed");
} catch (Exception e) {
LOG.info("Expected exception " + e.getMessage());
}
}
/**
* Test if writing to closed buffer fails.
*/
@Test
public void testCloseOutput() throws IOException {
LOG.info("Running test close output");
ByteArrayOutputStream os = new ByteArrayOutputStream(1000);
DataOutputStream dos = BufferedByteOutputStream.wrapOutputStream(os, 100,
10);
dos.close();
dos.close(); // can close multiple times
try {
// this will cause to flush BufferedOutputStream
for (int i = 0; i < 10000; i++) {
dos.write(1);
}
fail("Write should fail");
} catch (Exception e) {
LOG.info("Expected exception " + e.getMessage());
}
try {
// this will cause to flush BufferedOutputStream
for (int i = 0; i < 1000; i++) {
dos.write(new byte[10], 0, 10);
}
fail("Write should fail");
} catch (Exception e) {
LOG.info("Expected exception " + e.getMessage());
}
try {
dos.flush();
fail("Flush should fail");
} catch (Exception e) {
LOG.info("Expected exception " + e.getMessage());
}
}
private long testBuffer(final int inputSize, final int bufferSize,
final int fixedTempBufferSize, final boolean writeOutput)
throws Exception {
LOG.info("Running test raw buffer with input size: " + inputSize
+ ", buffer size: " + bufferSize);
setUp(inputSize);
final BufferedByteInputOutput buffer = new BufferedByteInputOutput(bufferSize);
Callable<Void> writerThread = new Callable<Void>() {
@Override
public Void call() throws Exception {
int totalWritten = 0, inputCursor = 0;
while (totalWritten < inputSize) {
if (rand.nextBoolean()) {
// write single byte
buffer.write(input[inputCursor++]);
totalWritten++;
} else {
int count;
if (fixedTempBufferSize > 0) {
count = Math.min(inputSize - totalWritten, rand.nextInt(fixedTempBufferSize) + 1);
} else {
count = rand.nextInt(inputSize - totalWritten) + 1;
}
buffer.write(input, inputCursor, count);
inputCursor += count;
totalWritten += count;
}
}
buffer.close();
return null;
}
};
Callable<Void> readerThread = new Callable<Void>() {
@Override
public Void call() throws Exception {
int totalRead = 0, outputCursor = 0;
while (totalRead < inputSize) {
if (rand.nextBoolean()) {
// read single byte
int b = buffer.read();
assertFalse(b == -1);
output[outputCursor++] = (byte) b;
totalRead++;
} else {
int count;
if (fixedTempBufferSize > 0) {
count = rand.nextInt(fixedTempBufferSize) + 1;
} else {
count = rand.nextInt(inputSize - totalRead) + 1;
}
byte[] bytes = new byte[count];
int bytesRead = buffer.read(bytes, 0, count);
assertFalse(bytesRead == -1);
if (writeOutput) {
System.arraycopy(bytes, 0, output, outputCursor, bytesRead);
outputCursor += bytesRead;
}
totalRead += bytesRead;
}
}
return null;
}
};
assertEquals(0, buffer.available());
//////////////// run writer and reader
long start = System.currentTimeMillis();
ExecutorService executor = Executors.newFixedThreadPool(2);
Future<Void> readerFuture = executor.submit(readerThread);
Future<Void> writerFuture = executor.submit(writerThread);
readerFuture.get();
writerFuture.get();
long stop = System.currentTimeMillis();
////////////////
if (writeOutput) {
assertTrue(Arrays.equals(input, output));
}
// check written and read bytes
assertEquals(inputSize, buffer.totalRead());
assertEquals(inputSize, buffer.totalWritten());
buffer.close();
// should get -1 after closing for reads
// since everything has been read
assertEquals(-1, buffer.read());
assertEquals(-1, buffer.read(new byte[10], 0, 10));
// should fail writes
try {
buffer.write(1);
fail("Should get exception after closing");
} catch (IOException e) {
LOG.info("Expected exception " + e.getMessage());
}
try {
buffer.write(new byte[10], 0, 10);
fail("Should get exception after closing");
} catch (IOException e) {
LOG.info("Expected exception " + e.getMessage());
}
assertTrue(buffer.isClosed());
return stop - start;
}
private void testInputStream(int inputSize, int bufferSize, int readBufferSize)
throws IOException {
LOG.info("Running test input stream with inputSize: " + inputSize
+ ", bufferSize: " + bufferSize + ", readBufferSize: " + readBufferSize);
setUp(inputSize);
ByteArrayInputStream is = new ByteArrayInputStream(input);
DataInputStream dis = BufferedByteInputStream.wrapInputStream(is,
bufferSize, readBufferSize);
int totalRead = 0;
int outputCursor = 0;
while (totalRead < inputSize) {
if (rand.nextBoolean()) {
// read single byte
output[outputCursor++] = dis.readByte();
totalRead++;
} else {
int count = rand.nextInt(inputSize - totalRead) + 1;
byte[] bytes = new byte[count];
int bytesRead = dis.read(bytes, 0, count);
System.arraycopy(bytes, 0, output, outputCursor, bytesRead);
outputCursor += bytesRead;
totalRead += bytesRead;
}
}
assertEquals(inputSize, totalRead);
assertTrue(Arrays.equals(input, output));
dis.close();
dis.close(); // multiple close should work
dis.close(); // multiple close should work
}
private void testOutputStream(int inputSize, int bufferSize,
int writeBufferSize, boolean closeAndCheck) throws IOException {
TestBufferedByteInputOutputInjectionHandler h =
new TestBufferedByteInputOutputInjectionHandler();
InjectionHandler.set(h);
LOG.info("Running test output stream with inputSize: " + inputSize
+ ", bufferSize: " + bufferSize + ", readBufferSize: "
+ writeBufferSize);
setUp(inputSize);
ByteArrayOutputStream os = new ByteArrayOutputStream(inputSize);
DataOutputStream dos = BufferedByteOutputStream.wrapOutputStream(os,
bufferSize, writeBufferSize);
int totalWritten = 0;
int inputCursor = 0;
while (totalWritten < inputSize) {
if (rand.nextBoolean()) {
// write single byte
dos.write(input[inputCursor++]);
totalWritten++;
} else {
int count = rand.nextInt(inputSize - totalWritten) + 1;
dos.write(input, inputCursor, count);
inputCursor += count;
totalWritten += count;
}
// random flush
if (rand.nextBoolean()) {
h.bytesFlushed = -1;
dos.flush();
assertEquals(totalWritten, h.bytesFlushed);
}
}
if (closeAndCheck) {
// we either close
dos.close();
} else {
// or need to flush the stream
dos.flush();
} // in either case data must be out
assertEquals(inputSize, totalWritten);
assertTrue(Arrays.equals(input, os.toByteArray()));
// close
dos.close();
dos.close(); // should be fine to call it multiple times
dos.close();
}
private int rand(int n) {
return rand.nextInt(n) + 1;
}
private void sleep(long ms) {
try {
Thread.sleep(ms);
} catch (InterruptedException e) {
LOG.info("Interrupted exception", e);
Thread.currentThread().interrupt();
}
}
class TestBufferedByteInputOutputInjectionHandler extends InjectionHandler {
volatile long bytesFlushed;
@Override
protected void _processEvent(InjectionEventI event, Object... args) {
if (event == InjectionEventCore.BUFFEREDBYTEOUTPUTSTREAM_FLUSH) {
bytesFlushed = (Long)args[0];
}
}
}
}