/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.io; import static org.junit.Assert.assertEquals; import java.io.IOException; import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.Test; public class TestSequenceFileSync { private static final int NUMRECORDS = 2000; private static final int RECORDSIZE = 80; private static final Random rand = new Random(); private final static String REC_FMT = "%d RECORDID %d : "; private static void forOffset(SequenceFile.Reader reader, IntWritable key, Text val, int iter, long off, int expectedRecord) throws IOException { val.clear(); reader.sync(off); reader.next(key, val); assertEquals(key.get(), expectedRecord); final String test = String.format(REC_FMT, expectedRecord, expectedRecord); assertEquals("Invalid value " + val, 0, val.find(test, 0)); } @Test public void testLowSyncpoint() throws IOException { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.getLocal(conf); final Path path = new Path(System.getProperty("test.build.data", "/tmp"), "sequencefile.sync.test"); final IntWritable input = new IntWritable(); final Text val = new Text(); SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, IntWritable.class, Text.class); try { writeSequenceFile(writer, NUMRECORDS); for (int i = 0; i < 5 ; i++) { final SequenceFile.Reader reader; //try different SequenceFile.Reader constructors if (i % 2 == 0) { reader = new SequenceFile.Reader(fs, path, conf); } else { final FSDataInputStream in = fs.open(path); final long length = fs.getFileStatus(path).getLen(); final int buffersize = conf.getInt("io.file.buffer.size", 4096); reader = new SequenceFile.Reader(in, buffersize, 0L, length, conf); } try { forOffset(reader, input, val, i, 0, 0); forOffset(reader, input, val, i, 65, 0); forOffset(reader, input, val, i, 2000, 21); forOffset(reader, input, val, i, 0, 0); } finally { reader.close(); } } } finally { fs.delete(path, false); } } public static void writeSequenceFile(SequenceFile.Writer writer, int numRecords) throws IOException { final IntWritable key = new IntWritable(); final Text val = new Text(); for (int numWritten = 0; numWritten < numRecords; ++numWritten) { key.set(numWritten); randomText(val, numWritten, RECORDSIZE); writer.append(key, val); } writer.close(); } static void randomText(Text val, int id, int recordSize) { val.clear(); final StringBuilder ret = new StringBuilder(recordSize); ret.append(String.format(REC_FMT, id, id)); recordSize -= ret.length(); for (int i = 0; i < recordSize; ++i) { ret.append(rand.nextInt(9)); } val.set(ret.toString()); } }