/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.contract;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.EOFException;
import java.io.IOException;
import java.util.Random;
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyRead;
/**
* Test Seek operations
*/
public abstract class AbstractContractSeekTest extends AbstractFSContractTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(AbstractContractSeekTest.class);
public static final int DEFAULT_RANDOM_SEEK_COUNT = 100;
private Path smallSeekFile;
private Path zeroByteFile;
private FSDataInputStream instream;
@Override
public void setup() throws Exception {
super.setup();
skipIfUnsupported(SUPPORTS_SEEK);
//delete the test directory
smallSeekFile = path("seekfile.txt");
zeroByteFile = path("zero.txt");
byte[] block = dataset(TEST_FILE_LEN, 0, 255);
//this file now has a simple rule: offset => value
FileSystem fs = getFileSystem();
createFile(fs, smallSeekFile, true, block);
touch(fs, zeroByteFile);
}
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
conf.setInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, 4096);
return conf;
}
@Override
public void teardown() throws Exception {
IOUtils.closeStream(instream);
instream = null;
super.teardown();
}
/**
* Skip a test case if the FS doesn't support positioned readable.
* This should hold automatically if the FS supports seek, even
* if it doesn't support seeking past the EOF.
* And, because this test suite requires seek to be supported, the
* feature is automatically assumed to be true unless stated otherwise.
*/
protected void assumeSupportsPositionedReadable() throws IOException {
// because this ,
if (!getContract().isSupported(SUPPORTS_POSITIONED_READABLE, true)) {
skip("Skipping as unsupported feature: "
+ SUPPORTS_POSITIONED_READABLE);
}
}
@Test
public void testSeekZeroByteFile() throws Throwable {
describe("seek and read a 0 byte file");
instream = getFileSystem().open(zeroByteFile);
assertEquals(0, instream.getPos());
//expect initial read to fai;
int result = instream.read();
assertMinusOne("initial byte read", result);
byte[] buffer = new byte[1];
//expect that seek to 0 works
instream.seek(0);
//reread, expect same exception
result = instream.read();
assertMinusOne("post-seek byte read", result);
result = instream.read(buffer, 0, 1);
assertMinusOne("post-seek buffer read", result);
}
@Test
public void testBlockReadZeroByteFile() throws Throwable {
describe("do a block read on a 0 byte file");
instream = getFileSystem().open(zeroByteFile);
assertEquals(0, instream.getPos());
//expect that seek to 0 works
byte[] buffer = new byte[1];
int result = instream.read(buffer, 0, 1);
assertMinusOne("block read zero byte file", result);
}
/**
* Seek and read on a closed file.
* Some filesystems let callers seek on a closed file -these must
* still fail on the subsequent reads.
* @throws Throwable
*/
@Test
public void testSeekReadClosedFile() throws Throwable {
instream = getFileSystem().open(smallSeekFile);
getLog().debug(
"Stream is of type " + instream.getClass().getCanonicalName());
instream.close();
try {
instream.seek(0);
if (!isSupported(SUPPORTS_SEEK_ON_CLOSED_FILE)) {
fail("seek succeeded on a closed stream");
}
} catch (IOException e) {
//expected a closed file
}
try {
int data = instream.available();
if (!isSupported(SUPPORTS_AVAILABLE_ON_CLOSED_FILE)) {
fail("available() succeeded on a closed stream, got " + data);
}
} catch (IOException e) {
//expected a closed file
}
try {
int data = instream.read();
fail("read() succeeded on a closed stream, got " + data);
} catch (IOException e) {
//expected a closed file
}
try {
byte[] buffer = new byte[1];
int result = instream.read(buffer, 0, 1);
fail("read(buffer, 0, 1) succeeded on a closed stream, got " + result);
} catch (IOException e) {
//expected a closed file
}
//what position does a closed file have?
try {
long offset = instream.getPos();
} catch (IOException e) {
// its valid to raise error here; but the test is applied to make
// sure there's no other exception like an NPE.
}
//and close again
instream.close();
}
@Test
public void testNegativeSeek() throws Throwable {
instream = getFileSystem().open(smallSeekFile);
assertEquals(0, instream.getPos());
try {
instream.seek(-1);
long p = instream.getPos();
LOG.warn("Seek to -1 returned a position of " + p);
int result = instream.read();
fail(
"expected an exception, got data " + result + " at a position of " + p);
} catch (EOFException e) {
//bad seek -expected
handleExpectedException(e);
} catch (IOException e) {
//bad seek -expected, but not as preferred as an EOFException
handleRelaxedException("a negative seek", "EOFException", e);
}
assertEquals(0, instream.getPos());
}
@Test
public void testSeekFile() throws Throwable {
describe("basic seek operations");
instream = getFileSystem().open(smallSeekFile);
assertEquals(0, instream.getPos());
//expect that seek to 0 works
instream.seek(0);
int result = instream.read();
assertEquals(0, result);
assertEquals(1, instream.read());
assertEquals(2, instream.getPos());
assertEquals(2, instream.read());
assertEquals(3, instream.getPos());
instream.seek(128);
assertEquals(128, instream.getPos());
assertEquals(128, instream.read());
instream.seek(63);
assertEquals(63, instream.read());
}
@Test
public void testSeekAndReadPastEndOfFile() throws Throwable {
describe("verify that reading past the last bytes in the file returns -1");
instream = getFileSystem().open(smallSeekFile);
assertEquals(0, instream.getPos());
//expect that seek to 0 works
//go just before the end
instream.seek(TEST_FILE_LEN - 2);
assertTrue("Premature EOF", instream.read() != -1);
assertTrue("Premature EOF", instream.read() != -1);
assertMinusOne("read past end of file", instream.read());
}
@Test
public void testSeekPastEndOfFileThenReseekAndRead() throws Throwable {
describe("do a seek past the EOF, then verify the stream recovers");
instream = getFileSystem().open(smallSeekFile);
//go just before the end. This may or may not fail; it may be delayed until the
//read
boolean canSeekPastEOF =
!getContract().isSupported(ContractOptions.REJECTS_SEEK_PAST_EOF, true);
try {
instream.seek(TEST_FILE_LEN + 1);
//if this doesn't trigger, then read() is expected to fail
assertMinusOne("read after seeking past EOF", instream.read());
} catch (EOFException e) {
//This is an error iff the FS claims to be able to seek past the EOF
if (canSeekPastEOF) {
//a failure wasn't expected
throw e;
}
handleExpectedException(e);
} catch (IOException e) {
//This is an error iff the FS claims to be able to seek past the EOF
if (canSeekPastEOF) {
//a failure wasn't expected
throw e;
}
handleRelaxedException("a seek past the end of the file",
"EOFException", e);
}
//now go back and try to read from a valid point in the file
instream.seek(1);
assertTrue("Premature EOF", instream.read() != -1);
}
/**
* Seek round a file bigger than IO buffers
* @throws Throwable
*/
@Test
public void testSeekBigFile() throws Throwable {
describe("Seek round a large file and verify the bytes are what is expected");
Path testSeekFile = path("bigseekfile.txt");
byte[] block = dataset(100 * 1024, 0, 255);
createFile(getFileSystem(), testSeekFile, false, block);
instream = getFileSystem().open(testSeekFile);
assertEquals(0, instream.getPos());
//expect that seek to 0 works
instream.seek(0);
int result = instream.read();
assertEquals(0, result);
assertEquals(1, instream.read());
assertEquals(2, instream.read());
//do seek 32KB ahead
instream.seek(32768);
assertEquals("@32768", block[32768], (byte) instream.read());
instream.seek(40000);
assertEquals("@40000", block[40000], (byte) instream.read());
instream.seek(8191);
assertEquals("@8191", block[8191], (byte) instream.read());
instream.seek(0);
assertEquals("@0", 0, (byte) instream.read());
// try read & readFully
instream.seek(0);
assertEquals(0, instream.getPos());
instream.read();
assertEquals(1, instream.getPos());
byte[] buf = new byte[80 * 1024];
instream.readFully(1, buf, 0, buf.length);
assertEquals(1, instream.getPos());
}
@Test
public void testPositionedBulkReadDoesntChangePosition() throws Throwable {
describe(
"verify that a positioned read does not change the getPos() value");
assumeSupportsPositionedReadable();
Path testSeekFile = path("bigseekfile.txt");
byte[] block = dataset(65536, 0, 255);
createFile(getFileSystem(), testSeekFile, false, block);
instream = getFileSystem().open(testSeekFile);
instream.seek(39999);
assertTrue(-1 != instream.read());
assertEquals(40000, instream.getPos());
int v = 256;
byte[] readBuffer = new byte[v];
assertEquals(v, instream.read(128, readBuffer, 0, v));
//have gone back
assertEquals(40000, instream.getPos());
//content is the same too
assertEquals("@40000", block[40000], (byte) instream.read());
//now verify the picked up data
for (int i = 0; i < 256; i++) {
assertEquals("@" + i, block[i + 128], readBuffer[i]);
}
}
/**
* Lifted from TestLocalFileSystem:
* Regression test for HADOOP-9307: BufferedFSInputStream returning
* wrong results after certain sequences of seeks and reads.
*/
@Test
public void testRandomSeeks() throws Throwable {
int limit = getContract().getLimit(TEST_RANDOM_SEEK_COUNT,
DEFAULT_RANDOM_SEEK_COUNT);
describe("Testing " + limit + " random seeks");
int filesize = 10 * 1024;
byte[] buf = dataset(filesize, 0, 255);
Path randomSeekFile = path("testrandomseeks.bin");
createFile(getFileSystem(), randomSeekFile, false, buf);
Random r = new Random();
// Record the sequence of seeks and reads which trigger a failure.
int[] seeks = new int[10];
int[] reads = new int[10];
try (FSDataInputStream stm = getFileSystem().open(randomSeekFile)) {
for (int i = 0; i < limit; i++) {
int seekOff = r.nextInt(buf.length);
int toRead = r.nextInt(Math.min(buf.length - seekOff, 32000));
seeks[i % seeks.length] = seekOff;
reads[i % reads.length] = toRead;
verifyRead(stm, buf, seekOff, toRead);
}
} catch (AssertionError afe) {
StringBuilder sb = new StringBuilder();
sb.append("Sequence of actions:\n");
for (int j = 0; j < seeks.length; j++) {
sb.append("seek @ ").append(seeks[j]).append(" ")
.append("read ").append(reads[j]).append("\n");
}
LOG.error(sb.toString());
throw afe;
}
}
@Test
public void testReadFullyZeroByteFile() throws Throwable {
describe("readFully against a 0 byte file");
assumeSupportsPositionedReadable();
instream = getFileSystem().open(zeroByteFile);
assertEquals(0, instream.getPos());
byte[] buffer = new byte[1];
instream.readFully(0, buffer, 0, 0);
assertEquals(0, instream.getPos());
// seek to 0 read 0 bytes from it
instream.seek(0);
assertEquals(0, instream.read(buffer, 0, 0));
}
@Test
public void testReadFullyPastEOFZeroByteFile() throws Throwable {
assumeSupportsPositionedReadable();
describe("readFully past the EOF of a 0 byte file");
instream = getFileSystem().open(zeroByteFile);
byte[] buffer = new byte[1];
// try to read past end of file
try {
instream.readFully(0, buffer, 0, 16);
fail("Expected an exception");
} catch (IllegalArgumentException | IndexOutOfBoundsException
| EOFException e) {
// expected
}
}
@Test
public void testReadFullySmallFile() throws Throwable {
describe("readFully operations");
assumeSupportsPositionedReadable();
instream = getFileSystem().open(smallSeekFile);
byte[] buffer = new byte[256];
// expect negative length to fail
try {
instream.readFully(0, buffer, 0, -16);
fail("Expected an exception");
} catch (IllegalArgumentException | IndexOutOfBoundsException e) {
// expected
}
// negative offset into buffer
try {
instream.readFully(0, buffer, -1, 16);
fail("Expected an exception");
} catch (IllegalArgumentException | IndexOutOfBoundsException e) {
// expected
}
// expect negative position to fail, ideally with EOF
try {
instream.readFully(-1, buffer);
fail("Expected an exception");
} catch (EOFException e) {
handleExpectedException(e);
} catch (IOException |IllegalArgumentException | IndexOutOfBoundsException e) {
handleRelaxedException("readFully with a negative position ",
"EOFException",
e);
}
// read more than the offset allows
try {
instream.readFully(0, buffer, buffer.length - 8, 16);
fail("Expected an exception");
} catch (IllegalArgumentException | IndexOutOfBoundsException e) {
// expected
}
// read properly
assertEquals(0, instream.getPos());
instream.readFully(0, buffer);
assertEquals(0, instream.getPos());
// now read the entire file in one go
byte[] fullFile = new byte[TEST_FILE_LEN];
instream.readFully(0, fullFile);
assertEquals(0, instream.getPos());
try {
instream.readFully(16, fullFile);
fail("Expected an exception");
} catch (EOFException e) {
handleExpectedException(e);
} catch (IOException e) {
handleRelaxedException("readFully which reads past EOF ",
"EOFException",
e);
}
}
@Test
public void testReadFullyPastEOF() throws Throwable {
describe("readFully past the EOF of a file");
assumeSupportsPositionedReadable();
instream = getFileSystem().open(smallSeekFile);
byte[] buffer = new byte[256];
// now read past the end of the file
try {
instream.readFully(TEST_FILE_LEN + 1, buffer);
fail("Expected an exception");
} catch (EOFException e) {
handleExpectedException(e);
} catch (IOException e) {
handleRelaxedException("readFully with an offset past EOF ",
"EOFException",
e);
}
// read zero bytes from an offset past EOF.
try {
instream.readFully(TEST_FILE_LEN + 1, buffer, 0, 0);
// a zero byte read may fail-fast
LOG.info("Filesystem short-circuits 0-byte reads");
} catch (EOFException e) {
handleExpectedException(e);
} catch (IOException e) {
handleRelaxedException("readFully(0 bytes) with an offset past EOF ",
"EOFException",
e);
}
}
@Test
public void testReadFullyZeroBytebufferPastEOF() throws Throwable {
describe("readFully zero bytes from an offset past EOF");
assumeSupportsPositionedReadable();
instream = getFileSystem().open(smallSeekFile);
byte[] buffer = new byte[256];
try {
instream.readFully(TEST_FILE_LEN + 1, buffer, 0, 0);
// a zero byte read may fail-fast
LOG.info("Filesystem short-circuits 0-byte reads");
} catch (EOFException e) {
handleExpectedException(e);
} catch (IOException e) {
handleRelaxedException("readFully(0 bytes) with an offset past EOF ",
"EOFException",
e);
}
}
@Test
public void testReadNullBuffer() throws Throwable {
describe("try to read a null buffer ");
assumeSupportsPositionedReadable();
try (FSDataInputStream in = getFileSystem().open(smallSeekFile)) {
// Null buffer
int r = in.read(0, null, 0, 16);
fail("Expected an exception from a read into a null buffer, got " + r);
} catch (IllegalArgumentException e) {
// expected
}
}
@Test
public void testReadSmallFile() throws Throwable {
describe("PositionedRead.read operations");
assumeSupportsPositionedReadable();
instream = getFileSystem().open(smallSeekFile);
byte[] buffer = new byte[256];
int r;
// expect negative length to fail
try {
r = instream.read(0, buffer, 0, -16);
fail("Expected an exception, got " + r);
} catch (IllegalArgumentException | IndexOutOfBoundsException e) {
// expected
}
// negative offset into buffer
try {
r = instream.read(0, buffer, -1, 16);
fail("Expected an exception, got " + r);
} catch (IllegalArgumentException | IndexOutOfBoundsException e) {
// expected
}
// negative position
try {
r = instream.read(-1, buffer, 0, 16);
fail("Expected an exception, got " + r);
} catch (EOFException e) {
handleExpectedException(e);
} catch (IOException | IllegalArgumentException | IndexOutOfBoundsException e) {
handleRelaxedException("read() with a negative position ",
"EOFException",
e);
}
// read more than the offset allows
try {
r = instream.read(0, buffer, buffer.length - 8, 16);
fail("Expected an exception, got " + r);
} catch (IllegalArgumentException | IndexOutOfBoundsException e) {
// expected
}
// read properly
assertEquals(0, instream.getPos());
instream.readFully(0, buffer);
assertEquals(0, instream.getPos());
// now read the entire file in one go
byte[] fullFile = new byte[TEST_FILE_LEN];
assertEquals(TEST_FILE_LEN,
instream.read(0, fullFile, 0, fullFile.length));
assertEquals(0, instream.getPos());
// now read past the end of the file
assertEquals(-1,
instream.read(TEST_FILE_LEN + 16, buffer, 0, 1));
}
@Test
public void testReadAtExactEOF() throws Throwable {
describe("read at the end of the file");
instream = getFileSystem().open(smallSeekFile);
instream.seek(TEST_FILE_LEN -1);
assertTrue("read at last byte", instream.read() > 0);
assertEquals("read just past EOF", -1, instream.read());
}
}