/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.client;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.NavigableSet;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.exceptions.ScannerResetException;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.hadoop.hbase.filter.LongComparator;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.WhileMatchFilter;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
import org.apache.hadoop.hbase.regionserver.DelegatingKeyValueScanner;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.ScanInfo;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
/**
* Run tests that use the HBase clients; {@link Table}.
* Sets up the HBase mini cluster once at start and runs through all client tests.
* Each creates a table named for the method and does its stuff against that.
*/
@Category({LargeTests.class, ClientTests.class})
@SuppressWarnings ("deprecation")
public class TestFromClientSide {
// NOTE: Increment tests were moved to their own class, TestIncrementsFromClientSide.
private static final Log LOG = LogFactory.getLog(TestFromClientSide.class);
protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static byte [] ROW = Bytes.toBytes("testRow");
private static byte [] FAMILY = Bytes.toBytes("testFamily");
private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
private static byte [] VALUE = Bytes.toBytes("testValue");
protected static int SLAVES = 3;
@Rule
public TestName name = new TestName();
/**
* @throws java.lang.Exception
*/
@BeforeClass
public static void setUpBeforeClass() throws Exception {
// Uncomment the following lines if more verbosity is needed for
// debugging (see HBASE-12285 for details).
//((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
//((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
//((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
Configuration conf = TEST_UTIL.getConfiguration();
conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
MultiRowMutationEndpoint.class.getName());
conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests
conf.setLong(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 6000000);
// We need more than one region server in this test
TEST_UTIL.startMiniCluster(SLAVES);
}
/**
* @throws java.lang.Exception
*/
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
/**
* @throws java.lang.Exception
*/
@Before
public void setUp() throws Exception {
// Nothing to do.
}
/**
* @throws java.lang.Exception
*/
@After
public void tearDown() throws Exception {
// Nothing to do.
}
/**
* Test append result when there are duplicate rpc request.
*/
@Test
public void testDuplicateAppend() throws Exception {
HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(name.getMethodName());
Map<String, String> kvs = new HashMap<>();
kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000");
hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1, kvs);
TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close();
Configuration c = new Configuration(TEST_UTIL.getConfiguration());
c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50);
// Client will retry beacuse rpc timeout is small than the sleep time of first rpc call
c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
Connection connection = ConnectionFactory.createConnection(c);
Table t = connection.getTable(TableName.valueOf(name.getMethodName()));
if (t instanceof HTable) {
HTable table = (HTable) t;
table.setOperationTimeout(3 * 1000);
try {
Append append = new Append(ROW);
append.add(TEST_UTIL.fam1, QUALIFIER, VALUE);
Result result = table.append(append);
// Verify expected result
Cell[] cells = result.rawCells();
assertEquals(1, cells.length);
assertKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, VALUE);
// Verify expected result again
Result readResult = table.get(new Get(ROW));
cells = readResult.rawCells();
assertEquals(1, cells.length);
assertKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, VALUE);
} finally {
table.close();
connection.close();
}
}
}
/**
* Basic client side validation of HBASE-4536
*/
@Test
public void testKeepDeletedCells() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
final byte[] FAMILY = Bytes.toBytes("family");
final byte[] C0 = Bytes.toBytes("c0");
final byte[] T1 = Bytes.toBytes("T1");
final byte[] T2 = Bytes.toBytes("T2");
final byte[] T3 = Bytes.toBytes("T3");
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
.setKeepDeletedCells(KeepDeletedCells.TRUE)
.setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE)
.setMaxVersions(3);
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(hcd);
TEST_UTIL.getAdmin().createTable(desc);
Table h = TEST_UTIL.getConnection().getTable(tableName);
long ts = System.currentTimeMillis();
Put p = new Put(T1, ts);
p.addColumn(FAMILY, C0, T1);
h.put(p);
p = new Put(T1, ts+2);
p.addColumn(FAMILY, C0, T2);
h.put(p);
p = new Put(T1, ts+4);
p.addColumn(FAMILY, C0, T3);
h.put(p);
Delete d = new Delete(T1, ts+3);
h.delete(d);
d = new Delete(T1, ts+3);
d.addColumns(FAMILY, C0, ts+3);
h.delete(d);
Get g = new Get(T1);
// does *not* include the delete
g.setTimeRange(0, ts+3);
Result r = h.get(g);
assertArrayEquals(T2, r.getValue(FAMILY, C0));
Scan s = new Scan(T1);
s.setTimeRange(0, ts+3);
s.setMaxVersions();
ResultScanner scanner = h.getScanner(s);
Cell[] kvs = scanner.next().rawCells();
assertArrayEquals(T2, CellUtil.cloneValue(kvs[0]));
assertArrayEquals(T1, CellUtil.cloneValue(kvs[1]));
scanner.close();
s = new Scan(T1);
s.setRaw(true);
s.setMaxVersions();
scanner = h.getScanner(s);
kvs = scanner.next().rawCells();
assertTrue(CellUtil.isDeleteFamily(kvs[0]));
assertArrayEquals(T3, CellUtil.cloneValue(kvs[1]));
assertTrue(CellUtil.isDelete(kvs[2]));
assertArrayEquals(T2, CellUtil.cloneValue(kvs[3]));
assertArrayEquals(T1, CellUtil.cloneValue(kvs[4]));
scanner.close();
h.close();
}
/**
* Basic client side validation of HBASE-10118
*/
@Test
public void testPurgeFutureDeletes() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
final byte[] ROW = Bytes.toBytes("row");
final byte[] FAMILY = Bytes.toBytes("family");
final byte[] COLUMN = Bytes.toBytes("column");
final byte[] VALUE = Bytes.toBytes("value");
Table table = TEST_UTIL.createTable(tableName, FAMILY);
// future timestamp
long ts = System.currentTimeMillis() * 2;
Put put = new Put(ROW, ts);
put.addColumn(FAMILY, COLUMN, VALUE);
table.put(put);
Get get = new Get(ROW);
Result result = table.get(get);
assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
Delete del = new Delete(ROW);
del.addColumn(FAMILY, COLUMN, ts);
table.delete(del);
get = new Get(ROW);
result = table.get(get);
assertNull(result.getValue(FAMILY, COLUMN));
// major compaction, purged future deletes
TEST_UTIL.getAdmin().flush(tableName);
TEST_UTIL.getAdmin().majorCompact(tableName);
// waiting for the major compaction to complete
TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
@Override
public boolean evaluate() throws IOException {
return TEST_UTIL.getAdmin().getCompactionState(tableName) ==
CompactionState.NONE;
}
});
put = new Put(ROW, ts);
put.addColumn(FAMILY, COLUMN, VALUE);
table.put(put);
get = new Get(ROW);
result = table.get(get);
assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
table.close();
}
/**
* Verifies that getConfiguration returns the same Configuration object used
* to create the HTable instance.
*/
@Test
public void testGetConfiguration() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
Configuration conf = TEST_UTIL.getConfiguration();
Table table = TEST_UTIL.createTable(tableName, FAMILIES);
assertSame(conf, table.getConfiguration());
}
/**
* Test from client side of an involved filter against a multi family that
* involves deletes.
*
* @throws Exception
*/
@Test
public void testWeirdCacheBehaviour() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
Table ht = TEST_UTIL.createTable(tableName, FAMILIES);
String value = "this is the value";
String value2 = "this is some other value";
String keyPrefix1 = UUID.randomUUID().toString();
String keyPrefix2 = UUID.randomUUID().toString();
String keyPrefix3 = UUID.randomUUID().toString();
putRows(ht, 3, value, keyPrefix1);
putRows(ht, 3, value, keyPrefix2);
putRows(ht, 3, value, keyPrefix3);
putRows(ht, 3, value2, keyPrefix1);
putRows(ht, 3, value2, keyPrefix2);
putRows(ht, 3, value2, keyPrefix3);
Table table = TEST_UTIL.getConnection().getTable(tableName);
System.out.println("Checking values for key: " + keyPrefix1);
assertEquals("Got back incorrect number of rows from scan", 3,
getNumberOfRows(keyPrefix1, value2, table));
System.out.println("Checking values for key: " + keyPrefix2);
assertEquals("Got back incorrect number of rows from scan", 3,
getNumberOfRows(keyPrefix2, value2, table));
System.out.println("Checking values for key: " + keyPrefix3);
assertEquals("Got back incorrect number of rows from scan", 3,
getNumberOfRows(keyPrefix3, value2, table));
deleteColumns(ht, value2, keyPrefix1);
deleteColumns(ht, value2, keyPrefix2);
deleteColumns(ht, value2, keyPrefix3);
System.out.println("Starting important checks.....");
assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
0, getNumberOfRows(keyPrefix1, value2, table));
assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
0, getNumberOfRows(keyPrefix2, value2, table));
assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
0, getNumberOfRows(keyPrefix3, value2, table));
}
private void deleteColumns(Table ht, String value, String keyPrefix)
throws IOException {
ResultScanner scanner = buildScanner(keyPrefix, value, ht);
Iterator<Result> it = scanner.iterator();
int count = 0;
while (it.hasNext()) {
Result result = it.next();
Delete delete = new Delete(result.getRow());
delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
ht.delete(delete);
count++;
}
assertEquals("Did not perform correct number of deletes", 3, count);
}
private int getNumberOfRows(String keyPrefix, String value, Table ht)
throws Exception {
ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
Iterator<Result> scanner = resultScanner.iterator();
int numberOfResults = 0;
while (scanner.hasNext()) {
Result result = scanner.next();
System.out.println("Got back key: " + Bytes.toString(result.getRow()));
for (Cell kv : result.rawCells()) {
System.out.println("kv=" + kv.toString() + ", "
+ Bytes.toString(CellUtil.cloneValue(kv)));
}
numberOfResults++;
}
return numberOfResults;
}
private ResultScanner buildScanner(String keyPrefix, String value, Table ht)
throws IOException {
// OurFilterList allFilters = new OurFilterList();
FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
.toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes
.toBytes(value));
filter.setFilterIfMissing(true);
allFilters.addFilter(filter);
// allFilters.addFilter(new
// RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
// Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes("trans-blob"));
scan.addFamily(Bytes.toBytes("trans-type"));
scan.addFamily(Bytes.toBytes("trans-date"));
scan.addFamily(Bytes.toBytes("trans-tags"));
scan.addFamily(Bytes.toBytes("trans-group"));
scan.setFilter(allFilters);
return ht.getScanner(scan);
}
private void putRows(Table ht, int numRows, String value, String key)
throws IOException {
for (int i = 0; i < numRows; i++) {
String row = key + "_" + UUID.randomUUID().toString();
System.out.println(String.format("Saving row: %s, with value %s", row,
value));
Put put = new Put(Bytes.toBytes(row));
put.setDurability(Durability.SKIP_WAL);
put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes
.toBytes("value for blob"));
put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
put.addColumn(Bytes.toBytes("trans-date"), null, Bytes
.toBytes("20090921010101999"));
put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
.toBytes(value));
put.addColumn(Bytes.toBytes("trans-group"), null, Bytes
.toBytes("adhocTransactionGroupId"));
ht.put(put);
}
}
/**
* Test filters when multiple regions. It does counts. Needs eye-balling of
* logs to ensure that we're not scanning more regions that we're supposed to.
* Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testFilterAcrossMultipleRegions()
throws IOException, InterruptedException {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table t = TEST_UTIL.createTable(tableName, FAMILY);
int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
assertRowCount(t, rowCount);
// Split the table. Should split on a reasonable key; 'lqj'
List<HRegionLocation> regions = splitTable(t);
assertRowCount(t, rowCount);
// Get end key of first region.
byte [] endKey = regions.get(0).getRegionInfo().getEndKey();
// Count rows with a filter that stops us before passed 'endKey'.
// Should be count of rows in first region.
int endKeyCount = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey));
assertTrue(endKeyCount < rowCount);
// How do I know I did not got to second region? Thats tough. Can't really
// do that in client-side region test. I verified by tracing in debugger.
// I changed the messages that come out when set to DEBUG so should see
// when scanner is done. Says "Finished with scanning..." with region name.
// Check that its finished in right region.
// New test. Make it so scan goes into next region by one and then two.
// Make sure count comes out right.
byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};
int plusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key));
assertEquals(endKeyCount + 1, plusOneCount);
key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};
int plusTwoCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key));
assertEquals(endKeyCount + 2, plusTwoCount);
// New test. Make it so I scan one less than endkey.
key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)};
int minusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key));
assertEquals(endKeyCount - 1, minusOneCount);
// For above test... study logs. Make sure we do "Finished with scanning.."
// in first region and that we do not fall into the next region.
key = new byte [] {'a', 'a', 'a'};
int countBBB = TEST_UTIL.countRows(t,
createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL));
assertEquals(1, countBBB);
int countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, null,
CompareFilter.CompareOp.GREATER_OR_EQUAL));
// Because started at start of table.
assertEquals(0, countGreater);
countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, endKey,
CompareFilter.CompareOp.GREATER_OR_EQUAL));
assertEquals(rowCount - endKeyCount, countGreater);
}
/**
* This is a coprocessor to inject a test failure so that a store scanner.reseek() call will
* fail with an IOException() on the first call.
*/
public static class ExceptionInReseekRegionObserver implements RegionObserver {
static AtomicLong reqCount = new AtomicLong(0);
static AtomicBoolean isDoNotRetry = new AtomicBoolean(false); // whether to throw DNRIOE
static AtomicBoolean throwOnce = new AtomicBoolean(true); // whether to only throw once
static void reset() {
reqCount.set(0);
isDoNotRetry.set(false);
throwOnce.set(true);
}
class MyStoreScanner extends StoreScanner {
public MyStoreScanner(Store store, ScanInfo scanInfo, Scan scan, NavigableSet<byte[]> columns,
long readPt) throws IOException {
super(store, scanInfo, scan, columns, readPt);
}
@Override
protected List<KeyValueScanner> selectScannersFrom(
List<? extends KeyValueScanner> allScanners) {
List<KeyValueScanner> scanners = super.selectScannersFrom(allScanners);
List<KeyValueScanner> newScanners = new ArrayList<>(scanners.size());
for (KeyValueScanner scanner : scanners) {
newScanners.add(new DelegatingKeyValueScanner(scanner) {
@Override
public boolean reseek(Cell key) throws IOException {
reqCount.incrementAndGet();
if (!throwOnce.get()|| reqCount.get() == 1) {
if (isDoNotRetry.get()) {
throw new DoNotRetryIOException("Injected exception");
} else {
throw new IOException("Injected exception");
}
}
return super.reseek(key);
}
});
}
return newScanners;
}
}
@Override
public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s,
final long readPt) throws IOException {
return new MyStoreScanner(store, store.getScanInfo(), scan, targetCols, readPt);
}
}
/**
* Tests the case where a Scan can throw an IOException in the middle of the seek / reseek
* leaving the server side RegionScanner to be in dirty state. The client has to ensure that the
* ClientScanner does not get an exception and also sees all the data.
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testClientScannerIsResetWhenScanThrowsIOException()
throws IOException, InterruptedException {
TEST_UTIL.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true);
final TableName tableName = TableName.valueOf(name.getMethodName());
HTableDescriptor htd = TEST_UTIL.createTableDescriptor(tableName, FAMILY);
htd.addCoprocessor(ExceptionInReseekRegionObserver.class.getName());
TEST_UTIL.getAdmin().createTable(htd);
ExceptionInReseekRegionObserver.reset();
ExceptionInReseekRegionObserver.throwOnce.set(true); // throw exceptions only once
try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
TEST_UTIL.getAdmin().flush(tableName);
int actualRowCount = TEST_UTIL.countRows(t, new Scan().addColumn(FAMILY, FAMILY));
assertEquals(rowCount, actualRowCount);
}
assertTrue(ExceptionInReseekRegionObserver.reqCount.get() > 0);
}
/**
* Tests the case where a coprocessor throws a DoNotRetryIOException in the scan. The expectation
* is that the exception will bubble up to the client scanner instead of being retried.
*/
@Test (timeout = 180000)
public void testScannerThrowsExceptionWhenCoprocessorThrowsDNRIOE()
throws IOException, InterruptedException {
TEST_UTIL.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true);
final TableName tableName = TableName.valueOf(name.getMethodName());
HTableDescriptor htd = TEST_UTIL.createTableDescriptor(tableName, FAMILY);
htd.addCoprocessor(ExceptionInReseekRegionObserver.class.getName());
TEST_UTIL.getAdmin().createTable(htd);
ExceptionInReseekRegionObserver.reset();
ExceptionInReseekRegionObserver.isDoNotRetry.set(true);
try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
TEST_UTIL.loadTable(t, FAMILY, false);
TEST_UTIL.getAdmin().flush(tableName);
TEST_UTIL.countRows(t, new Scan().addColumn(FAMILY, FAMILY));
fail("Should have thrown an exception");
} catch (DoNotRetryIOException expected) {
// expected
}
assertTrue(ExceptionInReseekRegionObserver.reqCount.get() > 0);
}
/**
* Tests the case where a coprocessor throws a regular IOException in the scan. The expectation
* is that the we will keep on retrying, but fail after the retries are exhausted instead of
* retrying indefinitely.
*/
@Test (timeout = 180000)
public void testScannerFailsAfterRetriesWhenCoprocessorThrowsIOE()
throws IOException, InterruptedException {
TEST_UTIL.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true);
final TableName tableName = TableName.valueOf(name.getMethodName());
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
HTableDescriptor htd = TEST_UTIL.createTableDescriptor(tableName, FAMILY);
htd.addCoprocessor(ExceptionInReseekRegionObserver.class.getName());
TEST_UTIL.getAdmin().createTable(htd);
ExceptionInReseekRegionObserver.reset();
ExceptionInReseekRegionObserver.throwOnce.set(false); // throw exceptions in every retry
try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
TEST_UTIL.loadTable(t, FAMILY, false);
TEST_UTIL.getAdmin().flush(tableName);
TEST_UTIL.countRows(t, new Scan().addColumn(FAMILY, FAMILY));
fail("Should have thrown an exception");
} catch (DoNotRetryIOException expected) {
assertTrue(expected instanceof ScannerResetException);
// expected
}
assertTrue(ExceptionInReseekRegionObserver.reqCount.get() >= 3);
}
/*
* @param key
* @return Scan with RowFilter that does LESS than passed key.
*/
private Scan createScanWithRowFilter(final byte [] key) {
return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS);
}
/*
* @param key
* @param op
* @param startRow
* @return Scan with RowFilter that does CompareOp op on passed key.
*/
private Scan createScanWithRowFilter(final byte [] key,
final byte [] startRow, CompareFilter.CompareOp op) {
// Make sure key is of some substance... non-null and > than first key.
assertTrue(key != null && key.length > 0 &&
Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0);
LOG.info("Key=" + Bytes.toString(key));
Scan s = startRow == null? new Scan(): new Scan(startRow);
Filter f = new RowFilter(op, new BinaryComparator(key));
f = new WhileMatchFilter(f);
s.setFilter(f);
return s;
}
private void assertRowCount(final Table t, final int expected)
throws IOException {
assertEquals(expected, TEST_UTIL.countRows(t, new Scan()));
}
/*
* Split table into multiple regions.
* @param t Table to split.
* @return Map of regions to servers.
* @throws IOException
*/
private List<HRegionLocation> splitTable(final Table t)
throws IOException, InterruptedException {
// Split this table in two.
Admin admin = TEST_UTIL.getAdmin();
admin.split(t.getName());
admin.close();
List<HRegionLocation> regions = waitOnSplit(t);
assertTrue(regions.size() > 1);
return regions;
}
/*
* Wait on table split. May return because we waited long enough on the split
* and it didn't happen. Caller should check.
* @param t
* @return Map of table regions; caller needs to check table actually split.
*/
private List<HRegionLocation> waitOnSplit(final Table t)
throws IOException {
try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(t.getName())) {
List<HRegionLocation> regions = locator.getAllRegionLocations();
int originalCount = regions.size();
for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) {
Thread.currentThread();
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
regions = locator.getAllRegionLocations();
if (regions.size() > originalCount)
break;
}
return regions;
}
}
@Test
public void testSuperSimple() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
Scan scan = new Scan();
scan.addColumn(FAMILY, tableName.toBytes());
ResultScanner scanner = ht.getScanner(scan);
Result result = scanner.next();
assertTrue("Expected null result", result == null);
scanner.close();
}
@Test
public void testMaxKeyValueSize() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Configuration conf = TEST_UTIL.getConfiguration();
String oldMaxSize = conf.get(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY);
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
byte[] value = new byte[4 * 1024 * 1024];
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, value);
ht.put(put);
try {
TEST_UTIL.getConfiguration().setInt(
ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, 2 * 1024 * 1024);
// Create new table so we pick up the change in Configuration.
try (Connection connection =
ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
try (Table t = connection.getTable(TableName.valueOf(FAMILY))) {
put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, value);
t.put(put);
}
}
fail("Inserting a too large KeyValue worked, should throw exception");
} catch(Exception e) {}
conf.set(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, oldMaxSize);
}
@Test
public void testFilters() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
byte [][] ROWS = makeN(ROW, 10);
byte [][] QUALIFIERS = {
Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
};
for(int i=0;i<10;i++) {
Put put = new Put(ROWS[i]);
put.setDurability(Durability.SKIP_WAL);
put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
ht.put(put);
}
Scan scan = new Scan();
scan.addFamily(FAMILY);
Filter filter = new QualifierFilter(CompareOp.EQUAL,
new RegexStringComparator("col[1-5]"));
scan.setFilter(filter);
ResultScanner scanner = ht.getScanner(scan);
int expectedIndex = 1;
for(Result result : ht.getScanner(scan)) {
assertEquals(result.size(), 1);
assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex]));
assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]),
QUALIFIERS[expectedIndex]));
expectedIndex++;
}
assertEquals(expectedIndex, 6);
scanner.close();
}
@Test
public void testFilterWithLongCompartor() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
byte [][] ROWS = makeN(ROW, 10);
byte [][] values = new byte[10][];
for (int i = 0; i < 10; i ++) {
values[i] = Bytes.toBytes(100L * i);
}
for(int i = 0; i < 10; i ++) {
Put put = new Put(ROWS[i]);
put.setDurability(Durability.SKIP_WAL);
put.addColumn(FAMILY, QUALIFIER, values[i]);
ht.put(put);
}
Scan scan = new Scan();
scan.addFamily(FAMILY);
Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.GREATER,
new LongComparator(500));
scan.setFilter(filter);
ResultScanner scanner = ht.getScanner(scan);
int expectedIndex = 0;
for(Result result : ht.getScanner(scan)) {
assertEquals(result.size(), 1);
assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500);
expectedIndex++;
}
assertEquals(expectedIndex, 4);
scanner.close();
}
@Test
public void testKeyOnlyFilter() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
byte [][] ROWS = makeN(ROW, 10);
byte [][] QUALIFIERS = {
Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
};
for(int i=0;i<10;i++) {
Put put = new Put(ROWS[i]);
put.setDurability(Durability.SKIP_WAL);
put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
ht.put(put);
}
Scan scan = new Scan();
scan.addFamily(FAMILY);
Filter filter = new KeyOnlyFilter(true);
scan.setFilter(filter);
ResultScanner scanner = ht.getScanner(scan);
int count = 0;
for(Result result : ht.getScanner(scan)) {
assertEquals(result.size(), 1);
assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT);
assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length);
count++;
}
assertEquals(count, 10);
scanner.close();
}
/**
* Test simple table and non-existent row cases.
*/
@Test
public void testSimpleMissing() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
byte [][] ROWS = makeN(ROW, 4);
// Try to get a row on an empty table
Get get = new Get(ROWS[0]);
Result result = ht.get(get);
assertEmptyResult(result);
get = new Get(ROWS[0]);
get.addFamily(FAMILY);
result = ht.get(get);
assertEmptyResult(result);
get = new Get(ROWS[0]);
get.addColumn(FAMILY, QUALIFIER);
result = ht.get(get);
assertEmptyResult(result);
Scan scan = new Scan();
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan(ROWS[0]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan(ROWS[0],ROWS[1]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan();
scan.addFamily(FAMILY);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan();
scan.addColumn(FAMILY, QUALIFIER);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Insert a row
Put put = new Put(ROWS[2]);
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
// Try to get empty rows around it
get = new Get(ROWS[1]);
result = ht.get(get);
assertEmptyResult(result);
get = new Get(ROWS[0]);
get.addFamily(FAMILY);
result = ht.get(get);
assertEmptyResult(result);
get = new Get(ROWS[3]);
get.addColumn(FAMILY, QUALIFIER);
result = ht.get(get);
assertEmptyResult(result);
// Try to scan empty rows around it
scan = new Scan(ROWS[3]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan(ROWS[0],ROWS[2]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Make sure we can actually get the row
get = new Get(ROWS[2]);
result = ht.get(get);
assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
get = new Get(ROWS[2]);
get.addFamily(FAMILY);
result = ht.get(get);
assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
get = new Get(ROWS[2]);
get.addColumn(FAMILY, QUALIFIER);
result = ht.get(get);
assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
// Make sure we can scan the row
scan = new Scan();
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
scan = new Scan(ROWS[0],ROWS[3]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
scan = new Scan(ROWS[2],ROWS[3]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
}
/**
* Test basic puts, gets, scans, and deletes for a single row
* in a multiple family table.
*/
@Test
public void testSingleRowMultipleFamily() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] ROWS = makeN(ROW, 3);
byte [][] FAMILIES = makeNAscii(FAMILY, 10);
byte [][] QUALIFIERS = makeN(QUALIFIER, 10);
byte [][] VALUES = makeN(VALUE, 10);
Table ht = TEST_UTIL.createTable(tableName, FAMILIES);
Get get;
Scan scan;
Delete delete;
Put put;
Result result;
////////////////////////////////////////////////////////////////////////////
// Insert one column to one family
////////////////////////////////////////////////////////////////////////////
put = new Put(ROWS[0]);
put.addColumn(FAMILIES[4], QUALIFIERS[0], VALUES[0]);
ht.put(put);
// Get the single column
getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
// Scan the single column
scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
// Get empty results around inserted column
getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
// Scan empty results around inserted column
scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
////////////////////////////////////////////////////////////////////////////
// Flush memstore and run same tests from storefiles
////////////////////////////////////////////////////////////////////////////
TEST_UTIL.flush();
// Redo get and scan tests from storefile
getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
////////////////////////////////////////////////////////////////////////////
// Now, Test reading from memstore and storefiles at once
////////////////////////////////////////////////////////////////////////////
// Insert multiple columns to two other families
put = new Put(ROWS[0]);
put.addColumn(FAMILIES[2], QUALIFIERS[2], VALUES[2]);
put.addColumn(FAMILIES[2], QUALIFIERS[4], VALUES[4]);
put.addColumn(FAMILIES[4], QUALIFIERS[4], VALUES[4]);
put.addColumn(FAMILIES[6], QUALIFIERS[6], VALUES[6]);
put.addColumn(FAMILIES[6], QUALIFIERS[7], VALUES[7]);
put.addColumn(FAMILIES[7], QUALIFIERS[7], VALUES[7]);
put.addColumn(FAMILIES[9], QUALIFIERS[0], VALUES[0]);
ht.put(put);
// Get multiple columns across multiple families and get empties around it
singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
// Scan multiple columns across multiple families and scan empties around it
singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
////////////////////////////////////////////////////////////////////////////
// Flush the table again
////////////////////////////////////////////////////////////////////////////
TEST_UTIL.flush();
// Redo tests again
singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
// Insert more data to memstore
put = new Put(ROWS[0]);
put.addColumn(FAMILIES[6], QUALIFIERS[5], VALUES[5]);
put.addColumn(FAMILIES[6], QUALIFIERS[8], VALUES[8]);
put.addColumn(FAMILIES[6], QUALIFIERS[9], VALUES[9]);
put.addColumn(FAMILIES[4], QUALIFIERS[3], VALUES[3]);
ht.put(put);
////////////////////////////////////////////////////////////////////////////
// Delete a storefile column
////////////////////////////////////////////////////////////////////////////
delete = new Delete(ROWS[0]);
delete.addColumns(FAMILIES[6], QUALIFIERS[7]);
ht.delete(delete);
// Try to get deleted column
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[6], QUALIFIERS[7]);
result = ht.get(get);
assertEmptyResult(result);
// Try to scan deleted column
scan = new Scan();
scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Make sure we can still get a column before it and after it
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[6], QUALIFIERS[6]);
result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[6], QUALIFIERS[8]);
result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
// Make sure we can still scan a column before it and after it
scan = new Scan();
scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
scan = new Scan();
scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
////////////////////////////////////////////////////////////////////////////
// Delete a memstore column
////////////////////////////////////////////////////////////////////////////
delete = new Delete(ROWS[0]);
delete.addColumns(FAMILIES[6], QUALIFIERS[8]);
ht.delete(delete);
// Try to get deleted column
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[6], QUALIFIERS[8]);
result = ht.get(get);
assertEmptyResult(result);
// Try to scan deleted column
scan = new Scan();
scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Make sure we can still get a column before it and after it
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[6], QUALIFIERS[6]);
result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[6], QUALIFIERS[9]);
result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
// Make sure we can still scan a column before it and after it
scan = new Scan();
scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
scan = new Scan();
scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
////////////////////////////////////////////////////////////////////////////
// Delete joint storefile/memstore family
////////////////////////////////////////////////////////////////////////////
delete = new Delete(ROWS[0]);
delete.addFamily(FAMILIES[4]);
ht.delete(delete);
// Try to get storefile column in deleted family
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[4], QUALIFIERS[4]);
result = ht.get(get);
assertEmptyResult(result);
// Try to get memstore column in deleted family
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[4], QUALIFIERS[3]);
result = ht.get(get);
assertEmptyResult(result);
// Try to get deleted family
get = new Get(ROWS[0]);
get.addFamily(FAMILIES[4]);
result = ht.get(get);
assertEmptyResult(result);
// Try to scan storefile column in deleted family
scan = new Scan();
scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Try to scan memstore column in deleted family
scan = new Scan();
scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Try to scan deleted family
scan = new Scan();
scan.addFamily(FAMILIES[4]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Make sure we can still get another family
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[2], QUALIFIERS[2]);
result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[6], QUALIFIERS[9]);
result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
// Make sure we can still scan another family
scan = new Scan();
scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
scan = new Scan();
scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
////////////////////////////////////////////////////////////////////////////
// Flush everything and rerun delete tests
////////////////////////////////////////////////////////////////////////////
TEST_UTIL.flush();
// Try to get storefile column in deleted family
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[4], QUALIFIERS[4]);
result = ht.get(get);
assertEmptyResult(result);
// Try to get memstore column in deleted family
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[4], QUALIFIERS[3]);
result = ht.get(get);
assertEmptyResult(result);
// Try to get deleted family
get = new Get(ROWS[0]);
get.addFamily(FAMILIES[4]);
result = ht.get(get);
assertEmptyResult(result);
// Try to scan storefile column in deleted family
scan = new Scan();
scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Try to scan memstore column in deleted family
scan = new Scan();
scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Try to scan deleted family
scan = new Scan();
scan.addFamily(FAMILIES[4]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Make sure we can still get another family
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[2], QUALIFIERS[2]);
result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[6], QUALIFIERS[9]);
result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
// Make sure we can still scan another family
scan = new Scan();
scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
scan = new Scan();
scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
}
@Test
public void testNull() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
// Null table name (should NOT work)
try {
TEST_UTIL.createTable((TableName)null, FAMILY);
fail("Creating a table with null name passed, should have failed");
} catch(Exception e) {}
// Null family (should NOT work)
try {
TEST_UTIL.createTable(tableName, new byte[][]{(byte[])null});
fail("Creating a table with a null family passed, should fail");
} catch(Exception e) {}
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
// Null row (should NOT work)
try {
Put put = new Put((byte[])null);
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
fail("Inserting a null row worked, should throw exception");
} catch(Exception e) {}
// Null qualifier (should work)
{
Put put = new Put(ROW);
put.addColumn(FAMILY, null, VALUE);
ht.put(put);
getTestNull(ht, ROW, FAMILY, VALUE);
scanTestNull(ht, ROW, FAMILY, VALUE);
Delete delete = new Delete(ROW);
delete.addColumns(FAMILY, null);
ht.delete(delete);
Get get = new Get(ROW);
Result result = ht.get(get);
assertEmptyResult(result);
}
// Use a new table
ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName() + "2"), FAMILY);
// Empty qualifier, byte[0] instead of null (should work)
try {
Put put = new Put(ROW);
put.addColumn(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
ht.put(put);
getTestNull(ht, ROW, FAMILY, VALUE);
scanTestNull(ht, ROW, FAMILY, VALUE);
// Flush and try again
TEST_UTIL.flush();
getTestNull(ht, ROW, FAMILY, VALUE);
scanTestNull(ht, ROW, FAMILY, VALUE);
Delete delete = new Delete(ROW);
delete.addColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
ht.delete(delete);
Get get = new Get(ROW);
Result result = ht.get(get);
assertEmptyResult(result);
} catch(Exception e) {
throw new IOException("Using a row with null qualifier threw exception, should ");
}
// Null value
try {
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, null);
ht.put(put);
Get get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
Result result = ht.get(get);
assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
Scan scan = new Scan();
scan.addColumn(FAMILY, QUALIFIER);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
Delete delete = new Delete(ROW);
delete.addColumns(FAMILY, QUALIFIER);
ht.delete(delete);
get = new Get(ROW);
result = ht.get(get);
assertEmptyResult(result);
} catch(Exception e) {
throw new IOException("Null values should be allowed, but threw exception");
}
}
@Test
public void testNullQualifier() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table table = TEST_UTIL.createTable(tableName, FAMILY);
// Work for Put
Put put = new Put(ROW);
put.addColumn(FAMILY, null, VALUE);
table.put(put);
// Work for Get, Scan
getTestNull(table, ROW, FAMILY, VALUE);
scanTestNull(table, ROW, FAMILY, VALUE);
// Work for Delete
Delete delete = new Delete(ROW);
delete.addColumns(FAMILY, null);
table.delete(delete);
Get get = new Get(ROW);
Result result = table.get(get);
assertEmptyResult(result);
// Work for Increment/Append
Increment increment = new Increment(ROW);
increment.addColumn(FAMILY, null, 1L);
table.increment(increment);
getTestNull(table, ROW, FAMILY, 1L);
table.incrementColumnValue(ROW, FAMILY, null, 1L);
getTestNull(table, ROW, FAMILY, 2L);
delete = new Delete(ROW);
delete.addColumns(FAMILY, null);
table.delete(delete);
Append append = new Append(ROW);
append.add(FAMILY, null, VALUE);
table.append(append);
getTestNull(table, ROW, FAMILY, VALUE);
// Work for checkAndMutate, checkAndPut, checkAndDelete
put = new Put(ROW);
put.addColumn(FAMILY, null, Bytes.toBytes("checkAndPut"));
table.put(put);
table.checkAndPut(ROW, FAMILY, null, VALUE, put);
RowMutations mutate = new RowMutations(ROW);
mutate.add(new Put(ROW).addColumn(FAMILY, null, Bytes.toBytes("checkAndMutate")));
table.checkAndMutate(ROW, FAMILY, null, CompareOp.EQUAL, Bytes.toBytes("checkAndPut"), mutate);
delete = new Delete(ROW);
delete.addColumns(FAMILY, null);
table.checkAndDelete(ROW, FAMILY, null, Bytes.toBytes("checkAndMutate"), delete);
}
@Test
public void testVersions() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
long [] STAMPS = makeStamps(20);
byte [][] VALUES = makeNAscii(VALUE, 20);
Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
// Insert 4 versions of same column
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
ht.put(put);
// Verify we can get each one properly
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
// Verify we don't accidentally get others
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
// Ensure maxVersions in query is respected
Get get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions(2);
Result result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[4], STAMPS[5]},
new byte[][] {VALUES[4], VALUES[5]},
0, 1);
Scan scan = new Scan(ROW);
scan.addColumn(FAMILY, QUALIFIER);
scan.setMaxVersions(2);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[4], STAMPS[5]},
new byte[][] {VALUES[4], VALUES[5]},
0, 1);
// Flush and redo
TEST_UTIL.flush();
// Verify we can get each one properly
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
// Verify we don't accidentally get others
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
// Ensure maxVersions in query is respected
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions(2);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[4], STAMPS[5]},
new byte[][] {VALUES[4], VALUES[5]},
0, 1);
scan = new Scan(ROW);
scan.addColumn(FAMILY, QUALIFIER);
scan.setMaxVersions(2);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[4], STAMPS[5]},
new byte[][] {VALUES[4], VALUES[5]},
0, 1);
// Add some memstore and retest
// Insert 4 more versions of same column and a dupe
put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
ht.put(put);
// Ensure maxVersions in query is respected
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions();
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
0, 7);
scan = new Scan(ROW);
scan.addColumn(FAMILY, QUALIFIER);
scan.setMaxVersions();
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
0, 7);
get = new Get(ROW);
get.setMaxVersions();
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
0, 7);
scan = new Scan(ROW);
scan.setMaxVersions();
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
0, 7);
// Verify we can get each one properly
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
// Verify we don't accidentally get others
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
// Ensure maxVersions of table is respected
TEST_UTIL.flush();
// Insert 4 more versions of same column and a dupe
put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
ht.put(put);
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
0, 9);
scan = new Scan(ROW);
scan.addColumn(FAMILY, QUALIFIER);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
0, 9);
// Delete a version in the memstore and a version in a storefile
Delete delete = new Delete(ROW);
delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
ht.delete(delete);
// Test that it's gone
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
0, 9);
scan = new Scan(ROW);
scan.addColumn(FAMILY, QUALIFIER);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
0, 9);
}
@Test
public void testVersionLimits() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] FAMILIES = makeNAscii(FAMILY, 3);
int [] LIMITS = {1,3,5};
long [] STAMPS = makeStamps(10);
byte [][] VALUES = makeNAscii(VALUE, 10);
Table ht = TEST_UTIL.createTable(tableName, FAMILIES, LIMITS);
// Insert limit + 1 on each family
Put put = new Put(ROW);
put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]);
put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]);
put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]);
put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]);
put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]);
put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]);
put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]);
put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]);
put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]);
put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]);
put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]);
put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]);
put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]);
ht.put(put);
// Verify we only get the right number out of each
// Family0
Get get = new Get(ROW);
get.addColumn(FAMILIES[0], QUALIFIER);
get.setMaxVersions(Integer.MAX_VALUE);
Result result = ht.get(get);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
new long [] {STAMPS[1]},
new byte[][] {VALUES[1]},
0, 0);
get = new Get(ROW);
get.addFamily(FAMILIES[0]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
new long [] {STAMPS[1]},
new byte[][] {VALUES[1]},
0, 0);
Scan scan = new Scan(ROW);
scan.addColumn(FAMILIES[0], QUALIFIER);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
new long [] {STAMPS[1]},
new byte[][] {VALUES[1]},
0, 0);
scan = new Scan(ROW);
scan.addFamily(FAMILIES[0]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
new long [] {STAMPS[1]},
new byte[][] {VALUES[1]},
0, 0);
// Family1
get = new Get(ROW);
get.addColumn(FAMILIES[1], QUALIFIER);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
0, 2);
get = new Get(ROW);
get.addFamily(FAMILIES[1]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
0, 2);
scan = new Scan(ROW);
scan.addColumn(FAMILIES[1], QUALIFIER);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
0, 2);
scan = new Scan(ROW);
scan.addFamily(FAMILIES[1]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
0, 2);
// Family2
get = new Get(ROW);
get.addColumn(FAMILIES[2], QUALIFIER);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
0, 4);
get = new Get(ROW);
get.addFamily(FAMILIES[2]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
0, 4);
scan = new Scan(ROW);
scan.addColumn(FAMILIES[2], QUALIFIER);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
0, 4);
scan = new Scan(ROW);
scan.addFamily(FAMILIES[2]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
0, 4);
// Try all families
get = new Get(ROW);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertTrue("Expected 9 keys but received " + result.size(),
result.size() == 9);
get = new Get(ROW);
get.addFamily(FAMILIES[0]);
get.addFamily(FAMILIES[1]);
get.addFamily(FAMILIES[2]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertTrue("Expected 9 keys but received " + result.size(),
result.size() == 9);
get = new Get(ROW);
get.addColumn(FAMILIES[0], QUALIFIER);
get.addColumn(FAMILIES[1], QUALIFIER);
get.addColumn(FAMILIES[2], QUALIFIER);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertTrue("Expected 9 keys but received " + result.size(),
result.size() == 9);
scan = new Scan(ROW);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertTrue("Expected 9 keys but received " + result.size(),
result.size() == 9);
scan = new Scan(ROW);
scan.setMaxVersions(Integer.MAX_VALUE);
scan.addFamily(FAMILIES[0]);
scan.addFamily(FAMILIES[1]);
scan.addFamily(FAMILIES[2]);
result = getSingleScanResult(ht, scan);
assertTrue("Expected 9 keys but received " + result.size(),
result.size() == 9);
scan = new Scan(ROW);
scan.setMaxVersions(Integer.MAX_VALUE);
scan.addColumn(FAMILIES[0], QUALIFIER);
scan.addColumn(FAMILIES[1], QUALIFIER);
scan.addColumn(FAMILIES[2], QUALIFIER);
result = getSingleScanResult(ht, scan);
assertTrue("Expected 9 keys but received " + result.size(),
result.size() == 9);
}
@Test
public void testDeleteFamilyVersion() throws Exception {
Admin admin = TEST_UTIL.getAdmin();
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 1);
byte [][] VALUES = makeN(VALUE, 5);
long [] ts = {1000, 2000, 3000, 4000, 5000};
Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5);
Put put = new Put(ROW);
for (int q = 0; q < 1; q++)
for (int t = 0; t < 5; t++)
put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
ht.put(put);
admin.flush(tableName);
Delete delete = new Delete(ROW);
delete.addFamilyVersion(FAMILY, ts[1]); // delete version '2000'
delete.addFamilyVersion(FAMILY, ts[3]); // delete version '4000'
ht.delete(delete);
admin.flush(tableName);
for (int i = 0; i < 1; i++) {
Get get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIERS[i]);
get.setMaxVersions(Integer.MAX_VALUE);
Result result = ht.get(get);
// verify version '1000'/'3000'/'5000' remains for all columns
assertNResult(result, ROW, FAMILY, QUALIFIERS[i],
new long [] {ts[0], ts[2], ts[4]},
new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
0, 2);
}
ht.close();
admin.close();
}
@Test
public void testDeleteFamilyVersionWithOtherDeletes() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 5);
byte [][] VALUES = makeN(VALUE, 5);
long [] ts = {1000, 2000, 3000, 4000, 5000};
Admin admin = TEST_UTIL.getAdmin();
Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5);
Put put = null;
Result result = null;
Get get = null;
Delete delete = null;
// 1. put on ROW
put = new Put(ROW);
for (int q = 0; q < 5; q++)
for (int t = 0; t < 5; t++)
put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
ht.put(put);
admin.flush(tableName);
// 2. put on ROWS[0]
byte [] ROW2 = Bytes.toBytes("myRowForTest");
put = new Put(ROW2);
for (int q = 0; q < 5; q++)
for (int t = 0; t < 5; t++)
put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
ht.put(put);
admin.flush(tableName);
// 3. delete on ROW
delete = new Delete(ROW);
// delete version <= 2000 of all columns
// note: addFamily must be the first since it will mask
// the subsequent other type deletes!
delete.addFamily(FAMILY, ts[1]);
// delete version '4000' of all columns
delete.addFamilyVersion(FAMILY, ts[3]);
// delete version <= 3000 of column 0
delete.addColumns(FAMILY, QUALIFIERS[0], ts[2]);
// delete version <= 5000 of column 2
delete.addColumns(FAMILY, QUALIFIERS[2], ts[4]);
// delete version 5000 of column 4
delete.addColumn(FAMILY, QUALIFIERS[4], ts[4]);
ht.delete(delete);
admin.flush(tableName);
// 4. delete on ROWS[0]
delete = new Delete(ROW2);
delete.addFamilyVersion(FAMILY, ts[1]); // delete version '2000'
delete.addFamilyVersion(FAMILY, ts[3]); // delete version '4000'
ht.delete(delete);
admin.flush(tableName);
// 5. check ROW
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIERS[0]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIERS[0],
new long [] {ts[4]},
new byte[][] {VALUES[4]},
0, 0);
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIERS[1]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIERS[1],
new long [] {ts[2], ts[4]},
new byte[][] {VALUES[2], VALUES[4]},
0, 1);
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIERS[2]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertEquals(0, result.size());
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIERS[3]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIERS[3],
new long [] {ts[2], ts[4]},
new byte[][] {VALUES[2], VALUES[4]},
0, 1);
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIERS[4]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIERS[4],
new long [] {ts[2]},
new byte[][] {VALUES[2]},
0, 0);
// 6. check ROWS[0]
for (int i = 0; i < 5; i++) {
get = new Get(ROW2);
get.addColumn(FAMILY, QUALIFIERS[i]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
// verify version '1000'/'3000'/'5000' remains for all columns
assertNResult(result, ROW2, FAMILY, QUALIFIERS[i],
new long [] {ts[0], ts[2], ts[4]},
new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
0, 2);
}
ht.close();
admin.close();
}
@Test
public void testDeleteWithFailed() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] ROWS = makeNAscii(ROW, 6);
byte [][] FAMILIES = makeNAscii(FAMILY, 3);
byte [][] VALUES = makeN(VALUE, 5);
long [] ts = {1000, 2000, 3000, 4000, 5000};
Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3);
Put put = new Put(ROW);
put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
ht.put(put);
// delete wrong family
Delete delete = new Delete(ROW);
delete.addFamily(FAMILIES[1], ts[0]);
ht.delete(delete);
Get get = new Get(ROW);
get.addFamily(FAMILIES[0]);
get.setMaxVersions(Integer.MAX_VALUE);
Result result = ht.get(get);
assertTrue(Bytes.equals(result.getValue(FAMILIES[0], QUALIFIER), VALUES[0]));
}
@Test
public void testDeletes() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] ROWS = makeNAscii(ROW, 6);
byte [][] FAMILIES = makeNAscii(FAMILY, 3);
byte [][] VALUES = makeN(VALUE, 5);
long [] ts = {1000, 2000, 3000, 4000, 5000};
Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3);
Put put = new Put(ROW);
put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
put.addColumn(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
ht.put(put);
Delete delete = new Delete(ROW);
delete.addFamily(FAMILIES[0], ts[0]);
ht.delete(delete);
Get get = new Get(ROW);
get.addFamily(FAMILIES[0]);
get.setMaxVersions(Integer.MAX_VALUE);
Result result = ht.get(get);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
new long [] {ts[1]},
new byte[][] {VALUES[1]},
0, 0);
Scan scan = new Scan(ROW);
scan.addFamily(FAMILIES[0]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
new long [] {ts[1]},
new byte[][] {VALUES[1]},
0, 0);
// Test delete latest version
put = new Put(ROW);
put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
put.addColumn(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
put.addColumn(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
put.addColumn(FAMILIES[0], null, ts[4], VALUES[4]);
put.addColumn(FAMILIES[0], null, ts[2], VALUES[2]);
put.addColumn(FAMILIES[0], null, ts[3], VALUES[3]);
ht.put(put);
delete = new Delete(ROW);
delete.addColumn(FAMILIES[0], QUALIFIER); // ts[4]
ht.delete(delete);
get = new Get(ROW);
get.addColumn(FAMILIES[0], QUALIFIER);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
new long [] {ts[1], ts[2], ts[3]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
0, 2);
scan = new Scan(ROW);
scan.addColumn(FAMILIES[0], QUALIFIER);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
new long [] {ts[1], ts[2], ts[3]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
0, 2);
// Test for HBASE-1847
delete = new Delete(ROW);
delete.addColumn(FAMILIES[0], null);
ht.delete(delete);
// Cleanup null qualifier
delete = new Delete(ROW);
delete.addColumns(FAMILIES[0], null);
ht.delete(delete);
// Expected client behavior might be that you can re-put deleted values
// But alas, this is not to be. We can't put them back in either case.
put = new Put(ROW);
put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
ht.put(put);
// It used to be due to the internal implementation of Get, that
// the Get() call would return ts[4] UNLIKE the Scan below. With
// the switch to using Scan for Get this is no longer the case.
get = new Get(ROW);
get.addFamily(FAMILIES[0]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
new long [] {ts[1], ts[2], ts[3]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
0, 2);
// The Scanner returns the previous values, the expected-naive-unexpected behavior
scan = new Scan(ROW);
scan.addFamily(FAMILIES[0]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
new long [] {ts[1], ts[2], ts[3]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
0, 2);
// Test deleting an entire family from one row but not the other various ways
put = new Put(ROWS[0]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
ht.put(put);
put = new Put(ROWS[1]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
ht.put(put);
put = new Put(ROWS[2]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
ht.put(put);
// Assert that above went in.
get = new Get(ROWS[2]);
get.addFamily(FAMILIES[1]);
get.addFamily(FAMILIES[2]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertTrue("Expected 4 key but received " + result.size() + ": " + result,
result.size() == 4);
delete = new Delete(ROWS[0]);
delete.addFamily(FAMILIES[2]);
ht.delete(delete);
delete = new Delete(ROWS[1]);
delete.addColumns(FAMILIES[1], QUALIFIER);
ht.delete(delete);
delete = new Delete(ROWS[2]);
delete.addColumn(FAMILIES[1], QUALIFIER);
delete.addColumn(FAMILIES[1], QUALIFIER);
delete.addColumn(FAMILIES[2], QUALIFIER);
ht.delete(delete);
get = new Get(ROWS[0]);
get.addFamily(FAMILIES[1]);
get.addFamily(FAMILIES[2]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertTrue("Expected 2 keys but received " + result.size(),
result.size() == 2);
assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
new long [] {ts[0], ts[1]},
new byte[][] {VALUES[0], VALUES[1]},
0, 1);
scan = new Scan(ROWS[0]);
scan.addFamily(FAMILIES[1]);
scan.addFamily(FAMILIES[2]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertTrue("Expected 2 keys but received " + result.size(),
result.size() == 2);
assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
new long [] {ts[0], ts[1]},
new byte[][] {VALUES[0], VALUES[1]},
0, 1);
get = new Get(ROWS[1]);
get.addFamily(FAMILIES[1]);
get.addFamily(FAMILIES[2]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertTrue("Expected 2 keys but received " + result.size(),
result.size() == 2);
scan = new Scan(ROWS[1]);
scan.addFamily(FAMILIES[1]);
scan.addFamily(FAMILIES[2]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertTrue("Expected 2 keys but received " + result.size(),
result.size() == 2);
get = new Get(ROWS[2]);
get.addFamily(FAMILIES[1]);
get.addFamily(FAMILIES[2]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertEquals(1, result.size());
assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
new long [] {ts[2]},
new byte[][] {VALUES[2]},
0, 0);
scan = new Scan(ROWS[2]);
scan.addFamily(FAMILIES[1]);
scan.addFamily(FAMILIES[2]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertEquals(1, result.size());
assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
new long [] {ts[2]},
new byte[][] {VALUES[2]},
0, 0);
// Test if we delete the family first in one row (HBASE-1541)
delete = new Delete(ROWS[3]);
delete.addFamily(FAMILIES[1]);
ht.delete(delete);
put = new Put(ROWS[3]);
put.addColumn(FAMILIES[2], QUALIFIER, VALUES[0]);
ht.put(put);
put = new Put(ROWS[4]);
put.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
put.addColumn(FAMILIES[2], QUALIFIER, VALUES[2]);
ht.put(put);
get = new Get(ROWS[3]);
get.addFamily(FAMILIES[1]);
get.addFamily(FAMILIES[2]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertTrue("Expected 1 key but received " + result.size(),
result.size() == 1);
get = new Get(ROWS[4]);
get.addFamily(FAMILIES[1]);
get.addFamily(FAMILIES[2]);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertTrue("Expected 2 keys but received " + result.size(),
result.size() == 2);
scan = new Scan(ROWS[3]);
scan.addFamily(FAMILIES[1]);
scan.addFamily(FAMILIES[2]);
scan.setMaxVersions(Integer.MAX_VALUE);
ResultScanner scanner = ht.getScanner(scan);
result = scanner.next();
assertTrue("Expected 1 key but received " + result.size(),
result.size() == 1);
assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
result = scanner.next();
assertTrue("Expected 2 keys but received " + result.size(),
result.size() == 2);
assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
scanner.close();
// Add test of bulk deleting.
for (int i = 0; i < 10; i++) {
byte [] bytes = Bytes.toBytes(i);
put = new Put(bytes);
put.setDurability(Durability.SKIP_WAL);
put.addColumn(FAMILIES[0], QUALIFIER, bytes);
ht.put(put);
}
for (int i = 0; i < 10; i++) {
byte [] bytes = Bytes.toBytes(i);
get = new Get(bytes);
get.addFamily(FAMILIES[0]);
result = ht.get(get);
assertTrue(result.size() == 1);
}
ArrayList<Delete> deletes = new ArrayList<>();
for (int i = 0; i < 10; i++) {
byte [] bytes = Bytes.toBytes(i);
delete = new Delete(bytes);
delete.addFamily(FAMILIES[0]);
deletes.add(delete);
}
ht.delete(deletes);
for (int i = 0; i < 10; i++) {
byte [] bytes = Bytes.toBytes(i);
get = new Get(bytes);
get.addFamily(FAMILIES[0]);
result = ht.get(get);
assertTrue(result.isEmpty());
}
}
/*
* Baseline "scalability" test.
*
* Tests one hundred families, one million columns, one million versions
*/
@Ignore @Test
public void testMillions() throws Exception {
// 100 families
// millions of columns
// millions of versions
}
@Ignore @Test
public void testMultipleRegionsAndBatchPuts() throws Exception {
// Two family table
// Insert lots of rows
// Insert to the same row with batched puts
// Insert to multiple rows with batched puts
// Split the table
// Get row from first region
// Get row from second region
// Scan all rows
// Insert to multiple regions with batched puts
// Get row from first region
// Get row from second region
// Scan all rows
}
@Ignore @Test
public void testMultipleRowMultipleFamily() throws Exception {
}
//
// JIRA Testers
//
/**
* HBASE-867
* If millions of columns in a column family, hbase scanner won't come up
*
* Test will create numRows rows, each with numColsPerRow columns
* (1 version each), and attempt to scan them all.
*
* To test at scale, up numColsPerRow to the millions
* (have not gotten that to work running as junit though)
*/
@Test
public void testJiraTest867() throws Exception {
int numRows = 10;
int numColsPerRow = 2000;
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] ROWS = makeN(ROW, numRows);
byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
// Insert rows
for(int i=0;i<numRows;i++) {
Put put = new Put(ROWS[i]);
put.setDurability(Durability.SKIP_WAL);
for(int j=0;j<numColsPerRow;j++) {
put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
}
assertTrue("Put expected to contain " + numColsPerRow + " columns but " +
"only contains " + put.size(), put.size() == numColsPerRow);
ht.put(put);
}
// Get a row
Get get = new Get(ROWS[numRows-1]);
Result result = ht.get(get);
assertNumKeys(result, numColsPerRow);
Cell [] keys = result.rawCells();
for(int i=0;i<result.size();i++) {
assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
}
// Scan the rows
Scan scan = new Scan();
ResultScanner scanner = ht.getScanner(scan);
int rowCount = 0;
while((result = scanner.next()) != null) {
assertNumKeys(result, numColsPerRow);
Cell [] kvs = result.rawCells();
for(int i=0;i<numColsPerRow;i++) {
assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
}
rowCount++;
}
scanner.close();
assertTrue("Expected to scan " + numRows + " rows but actually scanned "
+ rowCount + " rows", rowCount == numRows);
// flush and try again
TEST_UTIL.flush();
// Get a row
get = new Get(ROWS[numRows-1]);
result = ht.get(get);
assertNumKeys(result, numColsPerRow);
keys = result.rawCells();
for(int i=0;i<result.size();i++) {
assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
}
// Scan the rows
scan = new Scan();
scanner = ht.getScanner(scan);
rowCount = 0;
while((result = scanner.next()) != null) {
assertNumKeys(result, numColsPerRow);
Cell [] kvs = result.rawCells();
for(int i=0;i<numColsPerRow;i++) {
assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
}
rowCount++;
}
scanner.close();
assertTrue("Expected to scan " + numRows + " rows but actually scanned "
+ rowCount + " rows", rowCount == numRows);
}
/**
* HBASE-861
* get with timestamp will return a value if there is a version with an
* earlier timestamp
*/
@Test
public void testJiraTest861() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] VALUES = makeNAscii(VALUE, 7);
long [] STAMPS = makeStamps(7);
Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
// Insert three versions
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
ht.put(put);
// Get the middle value
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
// Try to get one version before (expect fail)
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
// Try to get one version after (expect fail)
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
// Try same from storefile
TEST_UTIL.flush();
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
// Insert two more versions surrounding others, into memstore
put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
ht.put(put);
// Check we can get everything we should and can't get what we shouldn't
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
// Try same from two storefiles
TEST_UTIL.flush();
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
}
/**
* HBASE-33
* Add a HTable get/obtainScanner method that retrieves all versions of a
* particular column and row between two timestamps
*/
@Test
public void testJiraTest33() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] VALUES = makeNAscii(VALUE, 7);
long [] STAMPS = makeStamps(7);
Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
// Insert lots versions
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
ht.put(put);
getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
// Try same from storefile
TEST_UTIL.flush();
getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
}
/**
* HBASE-1014
* commit(BatchUpdate) method should return timestamp
*/
@Test
public void testJiraTest1014() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
long manualStamp = 12345;
// Insert lots versions
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE);
ht.put(put);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp-1);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp+1);
}
/**
* HBASE-1182
* Scan for columns > some timestamp
*/
@Test
public void testJiraTest1182() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] VALUES = makeNAscii(VALUE, 7);
long [] STAMPS = makeStamps(7);
Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
// Insert lots versions
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
ht.put(put);
getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
// Try same from storefile
TEST_UTIL.flush();
getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
}
/**
* HBASE-52
* Add a means of scanning over all versions
*/
@Test
public void testJiraTest52() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [][] VALUES = makeNAscii(VALUE, 7);
long [] STAMPS = makeStamps(7);
Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
// Insert lots versions
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
ht.put(put);
getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
// Try same from storefile
TEST_UTIL.flush();
getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
}
//
// Bulk Testers
//
private void getVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
int start, int end)
throws IOException {
Get get = new Get(row);
get.addColumn(family, qualifier);
get.setMaxVersions(Integer.MAX_VALUE);
get.setTimeRange(stamps[start+1], Long.MAX_VALUE);
Result result = ht.get(get);
assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
}
private void getVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
throws IOException {
Get get = new Get(row);
get.addColumn(family, qualifier);
get.setMaxVersions(Integer.MAX_VALUE);
get.setTimeRange(stamps[start], stamps[end]+1);
Result result = ht.get(get);
assertNResult(result, row, family, qualifier, stamps, values, start, end);
}
private void getAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
throws IOException {
Get get = new Get(row);
get.addColumn(family, qualifier);
get.setMaxVersions(Integer.MAX_VALUE);
Result result = ht.get(get);
assertNResult(result, row, family, qualifier, stamps, values, start, end);
}
private void scanVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
int start, int end)
throws IOException {
Scan scan = new Scan(row);
scan.addColumn(family, qualifier);
scan.setMaxVersions(Integer.MAX_VALUE);
scan.setTimeRange(stamps[start+1], Long.MAX_VALUE);
Result result = getSingleScanResult(ht, scan);
assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
}
private void scanVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
throws IOException {
Scan scan = new Scan(row);
scan.addColumn(family, qualifier);
scan.setMaxVersions(Integer.MAX_VALUE);
scan.setTimeRange(stamps[start], stamps[end]+1);
Result result = getSingleScanResult(ht, scan);
assertNResult(result, row, family, qualifier, stamps, values, start, end);
}
private void scanAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
throws IOException {
Scan scan = new Scan(row);
scan.addColumn(family, qualifier);
scan.setMaxVersions(Integer.MAX_VALUE);
Result result = getSingleScanResult(ht, scan);
assertNResult(result, row, family, qualifier, stamps, values, start, end);
}
private void getVersionAndVerify(Table ht, byte [] row, byte [] family,
byte [] qualifier, long stamp, byte [] value)
throws Exception {
Get get = new Get(row);
get.addColumn(family, qualifier);
get.setTimeStamp(stamp);
get.setMaxVersions(Integer.MAX_VALUE);
Result result = ht.get(get);
assertSingleResult(result, row, family, qualifier, stamp, value);
}
private void getVersionAndVerifyMissing(Table ht, byte [] row, byte [] family,
byte [] qualifier, long stamp)
throws Exception {
Get get = new Get(row);
get.addColumn(family, qualifier);
get.setTimeStamp(stamp);
get.setMaxVersions(Integer.MAX_VALUE);
Result result = ht.get(get);
assertEmptyResult(result);
}
private void scanVersionAndVerify(Table ht, byte [] row, byte [] family,
byte [] qualifier, long stamp, byte [] value)
throws Exception {
Scan scan = new Scan(row);
scan.addColumn(family, qualifier);
scan.setTimeStamp(stamp);
scan.setMaxVersions(Integer.MAX_VALUE);
Result result = getSingleScanResult(ht, scan);
assertSingleResult(result, row, family, qualifier, stamp, value);
}
private void scanVersionAndVerifyMissing(Table ht, byte [] row,
byte [] family, byte [] qualifier, long stamp)
throws Exception {
Scan scan = new Scan(row);
scan.addColumn(family, qualifier);
scan.setTimeStamp(stamp);
scan.setMaxVersions(Integer.MAX_VALUE);
Result result = getSingleScanResult(ht, scan);
assertNullResult(result);
}
private void getTestNull(Table ht, byte [] row, byte [] family,
byte [] value)
throws Exception {
Get get = new Get(row);
get.addColumn(family, null);
Result result = ht.get(get);
assertSingleResult(result, row, family, null, value);
get = new Get(row);
get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
result = ht.get(get);
assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
get = new Get(row);
get.addFamily(family);
result = ht.get(get);
assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
get = new Get(row);
result = ht.get(get);
assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
}
private void getTestNull(Table ht, byte[] row, byte[] family, long value) throws Exception {
Get get = new Get(row);
get.addColumn(family, null);
Result result = ht.get(get);
assertSingleResult(result, row, family, null, value);
get = new Get(row);
get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
result = ht.get(get);
assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
get = new Get(row);
get.addFamily(family);
result = ht.get(get);
assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
get = new Get(row);
result = ht.get(get);
assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
}
private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value)
throws Exception {
scanTestNull(ht, row, family, value, false);
}
private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value,
boolean isReversedScan) throws Exception {
Scan scan = new Scan();
scan.setReversed(isReversedScan);
scan.addColumn(family, null);
Result result = getSingleScanResult(ht, scan);
assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
scan = new Scan();
scan.setReversed(isReversedScan);
scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
scan = new Scan();
scan.setReversed(isReversedScan);
scan.addFamily(family);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
scan = new Scan();
scan.setReversed(isReversedScan);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
}
private void singleRowGetTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
byte [][] QUALIFIERS, byte [][] VALUES)
throws Exception {
// Single column from memstore
Get get = new Get(ROWS[0]);
get.addColumn(FAMILIES[4], QUALIFIERS[0]);
Result result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
// Single column from storefile
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[2], QUALIFIERS[2]);
result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
// Single column from storefile, family match
get = new Get(ROWS[0]);
get.addFamily(FAMILIES[7]);
result = ht.get(get);
assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
// Two columns, one from memstore one from storefile, same family,
// wildcard match
get = new Get(ROWS[0]);
get.addFamily(FAMILIES[4]);
result = ht.get(get);
assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
FAMILIES[4], QUALIFIERS[4], VALUES[4]);
// Two columns, one from memstore one from storefile, same family,
// explicit match
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[4], QUALIFIERS[0]);
get.addColumn(FAMILIES[4], QUALIFIERS[4]);
result = ht.get(get);
assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
FAMILIES[4], QUALIFIERS[4], VALUES[4]);
// Three column, one from memstore two from storefile, different families,
// wildcard match
get = new Get(ROWS[0]);
get.addFamily(FAMILIES[4]);
get.addFamily(FAMILIES[7]);
result = ht.get(get);
assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
// Multiple columns from everywhere storefile, many family, wildcard
get = new Get(ROWS[0]);
get.addFamily(FAMILIES[2]);
get.addFamily(FAMILIES[4]);
get.addFamily(FAMILIES[6]);
get.addFamily(FAMILIES[7]);
result = ht.get(get);
assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
new int [][] {
{2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
});
// Multiple columns from everywhere storefile, many family, wildcard
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[2], QUALIFIERS[2]);
get.addColumn(FAMILIES[2], QUALIFIERS[4]);
get.addColumn(FAMILIES[4], QUALIFIERS[0]);
get.addColumn(FAMILIES[4], QUALIFIERS[4]);
get.addColumn(FAMILIES[6], QUALIFIERS[6]);
get.addColumn(FAMILIES[6], QUALIFIERS[7]);
get.addColumn(FAMILIES[7], QUALIFIERS[7]);
get.addColumn(FAMILIES[7], QUALIFIERS[8]);
result = ht.get(get);
assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
new int [][] {
{2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
});
// Everything
get = new Get(ROWS[0]);
result = ht.get(get);
assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
new int [][] {
{2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
});
// Get around inserted columns
get = new Get(ROWS[1]);
result = ht.get(get);
assertEmptyResult(result);
get = new Get(ROWS[0]);
get.addColumn(FAMILIES[4], QUALIFIERS[3]);
get.addColumn(FAMILIES[2], QUALIFIERS[3]);
result = ht.get(get);
assertEmptyResult(result);
}
private void singleRowScanTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
byte [][] QUALIFIERS, byte [][] VALUES)
throws Exception {
// Single column from memstore
Scan scan = new Scan();
scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
Result result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
// Single column from storefile
scan = new Scan();
scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
// Single column from storefile, family match
scan = new Scan();
scan.addFamily(FAMILIES[7]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
// Two columns, one from memstore one from storefile, same family,
// wildcard match
scan = new Scan();
scan.addFamily(FAMILIES[4]);
result = getSingleScanResult(ht, scan);
assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
FAMILIES[4], QUALIFIERS[4], VALUES[4]);
// Two columns, one from memstore one from storefile, same family,
// explicit match
scan = new Scan();
scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
result = getSingleScanResult(ht, scan);
assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
FAMILIES[4], QUALIFIERS[4], VALUES[4]);
// Three column, one from memstore two from storefile, different families,
// wildcard match
scan = new Scan();
scan.addFamily(FAMILIES[4]);
scan.addFamily(FAMILIES[7]);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
// Multiple columns from everywhere storefile, many family, wildcard
scan = new Scan();
scan.addFamily(FAMILIES[2]);
scan.addFamily(FAMILIES[4]);
scan.addFamily(FAMILIES[6]);
scan.addFamily(FAMILIES[7]);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
new int [][] {
{2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
});
// Multiple columns from everywhere storefile, many family, wildcard
scan = new Scan();
scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
scan.addColumn(FAMILIES[2], QUALIFIERS[4]);
scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
scan.addColumn(FAMILIES[7], QUALIFIERS[7]);
scan.addColumn(FAMILIES[7], QUALIFIERS[8]);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
new int [][] {
{2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
});
// Everything
scan = new Scan();
result = getSingleScanResult(ht, scan);
assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
new int [][] {
{2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
});
// Scan around inserted columns
scan = new Scan(ROWS[1]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan();
scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
scan.addColumn(FAMILIES[2], QUALIFIERS[3]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
}
/**
* Verify a single column using gets.
* Expects family and qualifier arrays to be valid for at least
* the range: idx-2 < idx < idx+2
*/
private void getVerifySingleColumn(Table ht,
byte [][] ROWS, int ROWIDX,
byte [][] FAMILIES, int FAMILYIDX,
byte [][] QUALIFIERS, int QUALIFIERIDX,
byte [][] VALUES, int VALUEIDX)
throws Exception {
Get get = new Get(ROWS[ROWIDX]);
Result result = ht.get(get);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
get = new Get(ROWS[ROWIDX]);
get.addFamily(FAMILIES[FAMILYIDX]);
result = ht.get(get);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
get = new Get(ROWS[ROWIDX]);
get.addFamily(FAMILIES[FAMILYIDX-2]);
get.addFamily(FAMILIES[FAMILYIDX]);
get.addFamily(FAMILIES[FAMILYIDX+2]);
result = ht.get(get);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
get = new Get(ROWS[ROWIDX]);
get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]);
result = ht.get(get);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
get = new Get(ROWS[ROWIDX]);
get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]);
get.addFamily(FAMILIES[FAMILYIDX]);
result = ht.get(get);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
get = new Get(ROWS[ROWIDX]);
get.addFamily(FAMILIES[FAMILYIDX]);
get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]);
get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]);
get.addFamily(FAMILIES[FAMILYIDX-1]);
get.addFamily(FAMILIES[FAMILYIDX+2]);
result = ht.get(get);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
}
/**
* Verify a single column using scanners.
* Expects family and qualifier arrays to be valid for at least
* the range: idx-2 to idx+2
* Expects row array to be valid for at least idx to idx+2
*/
private void scanVerifySingleColumn(Table ht,
byte [][] ROWS, int ROWIDX,
byte [][] FAMILIES, int FAMILYIDX,
byte [][] QUALIFIERS, int QUALIFIERIDX,
byte [][] VALUES, int VALUEIDX)
throws Exception {
Scan scan = new Scan();
Result result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
scan = new Scan(ROWS[ROWIDX]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
scan = new Scan();
scan.addFamily(FAMILIES[FAMILYIDX]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
scan = new Scan();
scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
scan = new Scan();
scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
scan.addFamily(FAMILIES[FAMILYIDX]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
scan = new Scan();
scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]);
scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
scan.addFamily(FAMILIES[FAMILYIDX+1]);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
}
/**
* Verify we do not read any values by accident around a single column
* Same requirements as getVerifySingleColumn
*/
private void getVerifySingleEmpty(Table ht,
byte [][] ROWS, int ROWIDX,
byte [][] FAMILIES, int FAMILYIDX,
byte [][] QUALIFIERS, int QUALIFIERIDX)
throws Exception {
Get get = new Get(ROWS[ROWIDX]);
get.addFamily(FAMILIES[4]);
get.addColumn(FAMILIES[4], QUALIFIERS[1]);
Result result = ht.get(get);
assertEmptyResult(result);
get = new Get(ROWS[ROWIDX]);
get.addFamily(FAMILIES[4]);
get.addColumn(FAMILIES[4], QUALIFIERS[2]);
result = ht.get(get);
assertEmptyResult(result);
get = new Get(ROWS[ROWIDX]);
get.addFamily(FAMILIES[3]);
get.addColumn(FAMILIES[4], QUALIFIERS[2]);
get.addFamily(FAMILIES[5]);
result = ht.get(get);
assertEmptyResult(result);
get = new Get(ROWS[ROWIDX+1]);
result = ht.get(get);
assertEmptyResult(result);
}
private void scanVerifySingleEmpty(Table ht,
byte [][] ROWS, int ROWIDX,
byte [][] FAMILIES, int FAMILYIDX,
byte [][] QUALIFIERS, int QUALIFIERIDX)
throws Exception {
Scan scan = new Scan(ROWS[ROWIDX+1]);
Result result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan();
scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
scan.addFamily(FAMILIES[FAMILYIDX-1]);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
}
//
// Verifiers
//
private void assertKey(Cell key, byte [] row, byte [] family,
byte [] qualifier, byte [] value)
throws Exception {
assertTrue("Expected row [" + Bytes.toString(row) + "] " +
"Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
equals(row, CellUtil.cloneRow(key)));
assertTrue("Expected family [" + Bytes.toString(family) + "] " +
"Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
equals(family, CellUtil.cloneFamily(key)));
assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
"Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
equals(qualifier, CellUtil.cloneQualifier(key)));
assertTrue("Expected value [" + Bytes.toString(value) + "] " +
"Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
equals(value, CellUtil.cloneValue(key)));
}
static void assertIncrementKey(Cell key, byte [] row, byte [] family,
byte [] qualifier, long value)
throws Exception {
assertTrue("Expected row [" + Bytes.toString(row) + "] " +
"Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
equals(row, CellUtil.cloneRow(key)));
assertTrue("Expected family [" + Bytes.toString(family) + "] " +
"Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
equals(family, CellUtil.cloneFamily(key)));
assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
"Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
equals(qualifier, CellUtil.cloneQualifier(key)));
assertTrue("Expected value [" + value + "] " +
"Got value [" + Bytes.toLong(CellUtil.cloneValue(key)) + "]",
Bytes.toLong(CellUtil.cloneValue(key)) == value);
}
private void assertNumKeys(Result result, int n) throws Exception {
assertTrue("Expected " + n + " keys but got " + result.size(),
result.size() == n);
}
private void assertNResult(Result result, byte [] row,
byte [][] families, byte [][] qualifiers, byte [][] values,
int [][] idxs)
throws Exception {
assertTrue("Expected row [" + Bytes.toString(row) + "] " +
"Got row [" + Bytes.toString(result.getRow()) +"]",
equals(row, result.getRow()));
assertTrue("Expected " + idxs.length + " keys but result contains "
+ result.size(), result.size() == idxs.length);
Cell [] keys = result.rawCells();
for(int i=0;i<keys.length;i++) {
byte [] family = families[idxs[i][0]];
byte [] qualifier = qualifiers[idxs[i][1]];
byte [] value = values[idxs[i][2]];
Cell key = keys[i];
byte[] famb = CellUtil.cloneFamily(key);
byte[] qualb = CellUtil.cloneQualifier(key);
byte[] valb = CellUtil.cloneValue(key);
assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
+ "] " + "Got family [" + Bytes.toString(famb) + "]",
equals(family, famb));
assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
+ "] " + "Got qualifier [" + Bytes.toString(qualb) + "]",
equals(qualifier, qualb));
assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
+ "Got value [" + Bytes.toString(valb) + "]",
equals(value, valb));
}
}
private void assertNResult(Result result, byte [] row,
byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
int start, int end)
throws IOException {
assertTrue("Expected row [" + Bytes.toString(row) + "] " +
"Got row [" + Bytes.toString(result.getRow()) +"]",
equals(row, result.getRow()));
int expectedResults = end - start + 1;
assertEquals(expectedResults, result.size());
Cell[] keys = result.rawCells();
for (int i=0; i<keys.length; i++) {
byte [] value = values[end-i];
long ts = stamps[end-i];
Cell key = keys[i];
assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
+ "] " + "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
CellUtil.matchingFamily(key, family));
assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
+ "] " + "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key))+ "]",
CellUtil.matchingQualifier(key, qualifier));
assertTrue("Expected ts [" + ts + "] " +
"Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp());
assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
+ "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
CellUtil.matchingValue(key, value));
}
}
/**
* Validate that result contains two specified keys, exactly.
* It is assumed key A sorts before key B.
*/
private void assertDoubleResult(Result result, byte [] row,
byte [] familyA, byte [] qualifierA, byte [] valueA,
byte [] familyB, byte [] qualifierB, byte [] valueB)
throws Exception {
assertTrue("Expected row [" + Bytes.toString(row) + "] " +
"Got row [" + Bytes.toString(result.getRow()) +"]",
equals(row, result.getRow()));
assertTrue("Expected two keys but result contains " + result.size(),
result.size() == 2);
Cell [] kv = result.rawCells();
Cell kvA = kv[0];
assertTrue("(A) Expected family [" + Bytes.toString(familyA) + "] " +
"Got family [" + Bytes.toString(CellUtil.cloneFamily(kvA)) + "]",
equals(familyA, CellUtil.cloneFamily(kvA)));
assertTrue("(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " +
"Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvA)) + "]",
equals(qualifierA, CellUtil.cloneQualifier(kvA)));
assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " +
"Got value [" + Bytes.toString(CellUtil.cloneValue(kvA)) + "]",
equals(valueA, CellUtil.cloneValue(kvA)));
Cell kvB = kv[1];
assertTrue("(B) Expected family [" + Bytes.toString(familyB) + "] " +
"Got family [" + Bytes.toString(CellUtil.cloneFamily(kvB)) + "]",
equals(familyB, CellUtil.cloneFamily(kvB)));
assertTrue("(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " +
"Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvB)) + "]",
equals(qualifierB, CellUtil.cloneQualifier(kvB)));
assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " +
"Got value [" + Bytes.toString(CellUtil.cloneValue(kvB)) + "]",
equals(valueB, CellUtil.cloneValue(kvB)));
}
private void assertSingleResult(Result result, byte [] row, byte [] family,
byte [] qualifier, byte [] value)
throws Exception {
assertTrue("Expected row [" + Bytes.toString(row) + "] " +
"Got row [" + Bytes.toString(result.getRow()) +"]",
equals(row, result.getRow()));
assertTrue("Expected a single key but result contains " + result.size(),
result.size() == 1);
Cell kv = result.rawCells()[0];
assertTrue("Expected family [" + Bytes.toString(family) + "] " +
"Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
equals(family, CellUtil.cloneFamily(kv)));
assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
"Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
equals(qualifier, CellUtil.cloneQualifier(kv)));
assertTrue("Expected value [" + Bytes.toString(value) + "] " +
"Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
equals(value, CellUtil.cloneValue(kv)));
}
private void assertSingleResult(Result result, byte[] row, byte[] family, byte[] qualifier,
long value) throws Exception {
assertTrue(
"Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow())
+ "]", equals(row, result.getRow()));
assertTrue("Expected a single key but result contains " + result.size(), result.size() == 1);
Cell kv = result.rawCells()[0];
assertTrue(
"Expected family [" + Bytes.toString(family) + "] " + "Got family ["
+ Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
equals(family, CellUtil.cloneFamily(kv)));
assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier ["
+ Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
equals(qualifier, CellUtil.cloneQualifier(kv)));
assertTrue(
"Expected value [" + value + "] " + "Got value [" + Bytes.toLong(CellUtil.cloneValue(kv))
+ "]", value == Bytes.toLong(CellUtil.cloneValue(kv)));
}
private void assertSingleResult(Result result, byte [] row, byte [] family,
byte [] qualifier, long ts, byte [] value)
throws Exception {
assertTrue("Expected row [" + Bytes.toString(row) + "] " +
"Got row [" + Bytes.toString(result.getRow()) +"]",
equals(row, result.getRow()));
assertTrue("Expected a single key but result contains " + result.size(),
result.size() == 1);
Cell kv = result.rawCells()[0];
assertTrue("Expected family [" + Bytes.toString(family) + "] " +
"Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
equals(family, CellUtil.cloneFamily(kv)));
assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
"Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
equals(qualifier, CellUtil.cloneQualifier(kv)));
assertTrue("Expected ts [" + ts + "] " +
"Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp());
assertTrue("Expected value [" + Bytes.toString(value) + "] " +
"Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
equals(value, CellUtil.cloneValue(kv)));
}
private void assertEmptyResult(Result result) throws Exception {
assertTrue("expected an empty result but result contains " +
result.size() + " keys", result.isEmpty());
}
private void assertNullResult(Result result) throws Exception {
assertTrue("expected null result but received a non-null result",
result == null);
}
//
// Helpers
//
private Result getSingleScanResult(Table ht, Scan scan) throws IOException {
ResultScanner scanner = ht.getScanner(scan);
Result result = scanner.next();
scanner.close();
return result;
}
private byte [][] makeNAscii(byte [] base, int n) {
if(n > 256) {
return makeNBig(base, n);
}
byte [][] ret = new byte[n][];
for(int i=0;i<n;i++) {
byte [] tail = Bytes.toBytes(Integer.toString(i));
ret[i] = Bytes.add(base, tail);
}
return ret;
}
private byte [][] makeN(byte [] base, int n) {
if (n > 256) {
return makeNBig(base, n);
}
byte [][] ret = new byte[n][];
for(int i=0;i<n;i++) {
ret[i] = Bytes.add(base, new byte[]{(byte)i});
}
return ret;
}
private byte [][] makeNBig(byte [] base, int n) {
byte [][] ret = new byte[n][];
for(int i=0;i<n;i++) {
int byteA = (i % 256);
int byteB = (i >> 8);
ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA});
}
return ret;
}
private long [] makeStamps(int n) {
long [] stamps = new long[n];
for(int i=0;i<n;i++) stamps[i] = i+1;
return stamps;
}
static boolean equals(byte [] left, byte [] right) {
if (left == null && right == null) return true;
if (left == null && right.length == 0) return true;
if (right == null && left.length == 0) return true;
return Bytes.equals(left, right);
}
@Test
public void testDuplicateVersions() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
long [] STAMPS = makeStamps(20);
byte [][] VALUES = makeNAscii(VALUE, 20);
Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
// Insert 4 versions of same column
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
ht.put(put);
// Verify we can get each one properly
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
// Verify we don't accidentally get others
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
// Ensure maxVersions in query is respected
Get get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions(2);
Result result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[4], STAMPS[5]},
new byte[][] {VALUES[4], VALUES[5]},
0, 1);
Scan scan = new Scan(ROW);
scan.addColumn(FAMILY, QUALIFIER);
scan.setMaxVersions(2);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[4], STAMPS[5]},
new byte[][] {VALUES[4], VALUES[5]},
0, 1);
// Flush and redo
TEST_UTIL.flush();
// Verify we can get each one properly
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
// Verify we don't accidentally get others
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
// Ensure maxVersions in query is respected
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions(2);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[4], STAMPS[5]},
new byte[][] {VALUES[4], VALUES[5]},
0, 1);
scan = new Scan(ROW);
scan.addColumn(FAMILY, QUALIFIER);
scan.setMaxVersions(2);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[4], STAMPS[5]},
new byte[][] {VALUES[4], VALUES[5]},
0, 1);
// Add some memstore and retest
// Insert 4 more versions of same column and a dupe
put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
ht.put(put);
// Ensure maxVersions in query is respected
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions(7);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
0, 6);
scan = new Scan(ROW);
scan.addColumn(FAMILY, QUALIFIER);
scan.setMaxVersions(7);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
0, 6);
get = new Get(ROW);
get.setMaxVersions(7);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
0, 6);
scan = new Scan(ROW);
scan.setMaxVersions(7);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
0, 6);
// Verify we can get each one properly
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
// Verify we don't accidentally get others
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
// Ensure maxVersions of table is respected
TEST_UTIL.flush();
// Insert 4 more versions of same column and a dupe
put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
ht.put(put);
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
0, 9);
scan = new Scan(ROW);
scan.addColumn(FAMILY, QUALIFIER);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
0, 9);
// Delete a version in the memstore and a version in a storefile
Delete delete = new Delete(ROW);
delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
ht.delete(delete);
// Test that it's gone
get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions(Integer.MAX_VALUE);
result = ht.get(get);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
0, 9);
scan = new Scan(ROW);
scan.addColumn(FAMILY, QUALIFIER);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILY, QUALIFIER,
new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
0, 9);
}
@Test
public void testUpdates() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
// Write a column with values at timestamp 1, 2 and 3
byte[] row = Bytes.toBytes("row1");
byte[] qualifier = Bytes.toBytes("myCol");
Put put = new Put(row);
put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
hTable.put(put);
put = new Put(row);
put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
hTable.put(put);
put = new Put(row);
put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
hTable.put(put);
Get get = new Get(row);
get.addColumn(FAMILY, qualifier);
get.setMaxVersions();
// Check that the column indeed has the right values at timestamps 1 and
// 2
Result result = hTable.get(get);
NavigableMap<Long, byte[]> navigableMap =
result.getMap().get(FAMILY).get(qualifier);
assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
// Update the value at timestamp 1
put = new Put(row);
put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
hTable.put(put);
// Update the value at timestamp 2
put = new Put(row);
put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
hTable.put(put);
// Check that the values at timestamp 2 and 1 got updated
result = hTable.get(get);
navigableMap = result.getMap().get(FAMILY).get(qualifier);
assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
}
@Test
public void testUpdatesWithMajorCompaction() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
Admin admin = TEST_UTIL.getAdmin();
// Write a column with values at timestamp 1, 2 and 3
byte[] row = Bytes.toBytes("row2");
byte[] qualifier = Bytes.toBytes("myCol");
Put put = new Put(row);
put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
hTable.put(put);
put = new Put(row);
put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
hTable.put(put);
put = new Put(row);
put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
hTable.put(put);
Get get = new Get(row);
get.addColumn(FAMILY, qualifier);
get.setMaxVersions();
// Check that the column indeed has the right values at timestamps 1 and
// 2
Result result = hTable.get(get);
NavigableMap<Long, byte[]> navigableMap =
result.getMap().get(FAMILY).get(qualifier);
assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
// Trigger a major compaction
admin.flush(tableName);
admin.majorCompact(tableName);
Thread.sleep(6000);
// Update the value at timestamp 1
put = new Put(row);
put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
hTable.put(put);
// Update the value at timestamp 2
put = new Put(row);
put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
hTable.put(put);
// Trigger a major compaction
admin.flush(tableName);
admin.majorCompact(tableName);
Thread.sleep(6000);
// Check that the values at timestamp 2 and 1 got updated
result = hTable.get(get);
navigableMap = result.getMap().get(FAMILY).get(qualifier);
assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
}
@Test
public void testMajorCompactionBetweenTwoUpdates() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
Admin admin = TEST_UTIL.getAdmin();
// Write a column with values at timestamp 1, 2 and 3
byte[] row = Bytes.toBytes("row3");
byte[] qualifier = Bytes.toBytes("myCol");
Put put = new Put(row);
put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
hTable.put(put);
put = new Put(row);
put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
hTable.put(put);
put = new Put(row);
put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
hTable.put(put);
Get get = new Get(row);
get.addColumn(FAMILY, qualifier);
get.setMaxVersions();
// Check that the column indeed has the right values at timestamps 1 and
// 2
Result result = hTable.get(get);
NavigableMap<Long, byte[]> navigableMap =
result.getMap().get(FAMILY).get(qualifier);
assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
// Trigger a major compaction
admin.flush(tableName);
admin.majorCompact(tableName);
Thread.sleep(6000);
// Update the value at timestamp 1
put = new Put(row);
put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
hTable.put(put);
// Trigger a major compaction
admin.flush(tableName);
admin.majorCompact(tableName);
Thread.sleep(6000);
// Update the value at timestamp 2
put = new Put(row);
put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
hTable.put(put);
// Trigger a major compaction
admin.flush(tableName);
admin.majorCompact(tableName);
Thread.sleep(6000);
// Check that the values at timestamp 2 and 1 got updated
result = hTable.get(get);
navigableMap = result.getMap().get(FAMILY).get(qualifier);
assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
}
@Test
public void testGet_EmptyTable() throws IOException {
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
Get get = new Get(ROW);
get.addFamily(FAMILY);
Result r = table.get(get);
assertTrue(r.isEmpty());
}
@Test
public void testGet_NullQualifier() throws IOException {
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, VALUE);
table.put(put);
put = new Put(ROW);
put.addColumn(FAMILY, null, VALUE);
table.put(put);
LOG.info("Row put");
Get get = new Get(ROW);
get.addColumn(FAMILY, null);
Result r = table.get(get);
assertEquals(1, r.size());
get = new Get(ROW);
get.addFamily(FAMILY);
r = table.get(get);
assertEquals(2, r.size());
}
@Test
public void testGet_NonExistentRow() throws IOException {
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, VALUE);
table.put(put);
LOG.info("Row put");
Get get = new Get(ROW);
get.addFamily(FAMILY);
Result r = table.get(get);
assertFalse(r.isEmpty());
System.out.println("Row retrieved successfully");
byte [] missingrow = Bytes.toBytes("missingrow");
get = new Get(missingrow);
get.addFamily(FAMILY);
r = table.get(get);
assertTrue(r.isEmpty());
LOG.info("Row missing as it should be");
}
@Test
public void testPut() throws IOException {
final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
final byte [] row1 = Bytes.toBytes("row1");
final byte [] row2 = Bytes.toBytes("row2");
final byte [] value = Bytes.toBytes("abcd");
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
Put put = new Put(row1);
put.addColumn(CONTENTS_FAMILY, null, value);
table.put(put);
put = new Put(row2);
put.addColumn(CONTENTS_FAMILY, null, value);
assertEquals(put.size(), 1);
assertEquals(put.getFamilyCellMap().get(CONTENTS_FAMILY).size(), 1);
// KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO
KeyValue kv = (KeyValue)put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY));
// will it return null or an empty byte array?
assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0]));
assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value));
table.put(put);
Scan scan = new Scan();
scan.addColumn(CONTENTS_FAMILY, null);
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
for(Cell key : r.rawCells()) {
System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
}
}
}
@Test
public void testPutNoCF() throws IOException {
final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
final byte[] VAL = Bytes.toBytes(100);
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
boolean caughtNSCFE = false;
try {
Put p = new Put(ROW);
p.addColumn(BAD_FAM, QUALIFIER, VAL);
table.put(p);
} catch (RetriesExhaustedWithDetailsException e) {
caughtNSCFE = e.getCause(0) instanceof NoSuchColumnFamilyException;
}
assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
}
@Test
public void testRowsPut() throws IOException {
final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
final int NB_BATCH_ROWS = 10;
final byte[] value = Bytes.toBytes("abcd");
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
ArrayList<Put> rowsUpdate = new ArrayList<Put>();
for (int i = 0; i < NB_BATCH_ROWS; i++) {
byte[] row = Bytes.toBytes("row" + i);
Put put = new Put(row);
put.setDurability(Durability.SKIP_WAL);
put.addColumn(CONTENTS_FAMILY, null, value);
rowsUpdate.add(put);
}
table.put(rowsUpdate);
Scan scan = new Scan();
scan.addFamily(CONTENTS_FAMILY);
ResultScanner scanner = table.getScanner(scan);
int nbRows = 0;
for (@SuppressWarnings("unused")
Result row : scanner)
nbRows++;
assertEquals(NB_BATCH_ROWS, nbRows);
}
@Test
public void testRowsPutBufferedManyManyFlushes() throws IOException {
final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
final byte[] value = Bytes.toBytes("abcd");
final int NB_BATCH_ROWS = 10;
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
table.setWriteBufferSize(10);
ArrayList<Put> rowsUpdate = new ArrayList<Put>();
for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
byte[] row = Bytes.toBytes("row" + i);
Put put = new Put(row);
put.setDurability(Durability.SKIP_WAL);
put.addColumn(CONTENTS_FAMILY, null, value);
rowsUpdate.add(put);
}
table.put(rowsUpdate);
Scan scan = new Scan();
scan.addFamily(CONTENTS_FAMILY);
ResultScanner scanner = table.getScanner(scan);
int nbRows = 0;
for (@SuppressWarnings("unused")
Result row : scanner)
nbRows++;
assertEquals(NB_BATCH_ROWS * 10, nbRows);
}
@Test
public void testAddKeyValue() throws IOException {
final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
final byte[] value = Bytes.toBytes("abcd");
final byte[] row1 = Bytes.toBytes("row1");
final byte[] row2 = Bytes.toBytes("row2");
byte[] qualifier = Bytes.toBytes("qf1");
Put put = new Put(row1);
// Adding KeyValue with the same row
KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
boolean ok = true;
try {
put.add(kv);
} catch (IOException e) {
ok = false;
}
assertEquals(true, ok);
// Adding KeyValue with the different row
kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
ok = false;
try {
put.add(kv);
} catch (IOException e) {
ok = true;
}
assertEquals(true, ok);
}
/**
* test for HBASE-737
* @throws IOException
*/
@Test
public void testHBase737 () throws IOException {
final byte [] FAM1 = Bytes.toBytes("fam1");
final byte [] FAM2 = Bytes.toBytes("fam2");
// Open table
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
new byte [][] {FAM1, FAM2});
// Insert some values
Put put = new Put(ROW);
put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
table.put(put);
try {
Thread.sleep(1000);
} catch (InterruptedException i) {
//ignore
}
put = new Put(ROW);
put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
table.put(put);
try {
Thread.sleep(1000);
} catch (InterruptedException i) {
//ignore
}
put = new Put(ROW);
put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
table.put(put);
long times[] = new long[3];
// First scan the memstore
Scan scan = new Scan();
scan.addFamily(FAM1);
scan.addFamily(FAM2);
ResultScanner s = table.getScanner(scan);
try {
int index = 0;
Result r = null;
while ((r = s.next()) != null) {
for(Cell key : r.rawCells()) {
times[index++] = key.getTimestamp();
}
}
} finally {
s.close();
}
for (int i = 0; i < times.length - 1; i++) {
for (int j = i + 1; j < times.length; j++) {
assertTrue(times[j] > times[i]);
}
}
// Flush data to disk and try again
TEST_UTIL.flush();
// Reset times
for(int i=0;i<times.length;i++) {
times[i] = 0;
}
try {
Thread.sleep(1000);
} catch (InterruptedException i) {
//ignore
}
scan = new Scan();
scan.addFamily(FAM1);
scan.addFamily(FAM2);
s = table.getScanner(scan);
try {
int index = 0;
Result r = null;
while ((r = s.next()) != null) {
for(Cell key : r.rawCells()) {
times[index++] = key.getTimestamp();
}
}
} finally {
s.close();
}
for (int i = 0; i < times.length - 1; i++) {
for (int j = i + 1; j < times.length; j++) {
assertTrue(times[j] > times[i]);
}
}
}
@Test
public void testListTables() throws IOException, InterruptedException {
final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
final TableName tableName3 = TableName.valueOf(name.getMethodName() + "3");
TableName [] tables = new TableName[] { tableName1, tableName2, tableName3 };
for (int i = 0; i < tables.length; i++) {
TEST_UTIL.createTable(tables[i], FAMILY);
}
Admin admin = TEST_UTIL.getAdmin();
HTableDescriptor[] ts = admin.listTables();
HashSet<HTableDescriptor> result = new HashSet<HTableDescriptor>(ts.length);
Collections.addAll(result, ts);
int size = result.size();
assertTrue(size >= tables.length);
for (int i = 0; i < tables.length && i < size; i++) {
boolean found = false;
for (int j = 0; j < ts.length; j++) {
if (ts[j].getTableName().equals(tables[i])) {
found = true;
break;
}
}
assertTrue("Not found: " + tables[i], found);
}
}
/**
* simple test that just executes parts of the client
* API that accept a pre-created Connection instance
*
* @throws IOException
*/
@Test
public void testUnmanagedHConnection() throws IOException {
final TableName tableName = TableName.valueOf(name.getMethodName());
TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
Table t = conn.getTable(tableName);
Admin admin = conn.getAdmin();
assertTrue(admin.tableExists(tableName));
assertTrue(t.get(new Get(ROW)).isEmpty());
admin.close();
}
/**
* test of that unmanaged HConnections are able to reconnect
* properly (see HBASE-5058)
*
* @throws Exception
*/
@Test
public void testUnmanagedHConnectionReconnect() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
Table t = conn.getTable(tableName);
try (Admin admin = conn.getAdmin()) {
assertTrue(admin.tableExists(tableName));
assertTrue(t.get(new Get(ROW)).isEmpty());
}
// stop the master
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
cluster.stopMaster(0, false);
cluster.waitOnMaster(0);
// start up a new master
cluster.startMaster();
assertTrue(cluster.waitForActiveAndReadyMaster());
// test that the same unmanaged connection works with a new
// Admin and can connect to the new master;
try (Admin admin = conn.getAdmin()) {
assertTrue(admin.tableExists(tableName));
assertTrue(admin.getClusterStatus().getServersSize() == SLAVES + 1);
}
}
@Test
public void testMiscHTableStuff() throws IOException {
final TableName tableAname = TableName.valueOf(name.getMethodName() + "A");
final TableName tableBname = TableName.valueOf(name.getMethodName() + "B");
final byte[] attrName = Bytes.toBytes("TESTATTR");
final byte[] attrValue = Bytes.toBytes("somevalue");
byte[] value = Bytes.toBytes("value");
Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY);
Put put = new Put(ROW);
put.addColumn(HConstants.CATALOG_FAMILY, null, value);
a.put(put);
// open a new connection to A and a connection to b
Table newA = TEST_UTIL.getConnection().getTable(tableAname);
// copy data from A to B
Scan scan = new Scan();
scan.addFamily(HConstants.CATALOG_FAMILY);
ResultScanner s = newA.getScanner(scan);
try {
for (Result r : s) {
put = new Put(r.getRow());
put.setDurability(Durability.SKIP_WAL);
for (Cell kv : r.rawCells()) {
put.add(kv);
}
b.put(put);
}
} finally {
s.close();
}
// Opening a new connection to A will cause the tables to be reloaded
Table anotherA = TEST_UTIL.getConnection().getTable(tableAname);
Get get = new Get(ROW);
get.addFamily(HConstants.CATALOG_FAMILY);
anotherA.get(get);
// We can still access A through newA because it has the table information
// cached. And if it needs to recalibrate, that will cause the information
// to be reloaded.
// Test user metadata
Admin admin = TEST_UTIL.getAdmin();
// make a modifiable descriptor
HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor());
// offline the table
admin.disableTable(tableAname);
// add a user attribute to HTD
desc.setValue(attrName, attrValue);
// add a user attribute to HCD
for (HColumnDescriptor c : desc.getFamilies())
c.setValue(attrName, attrValue);
// update metadata for all regions of this table
admin.modifyTable(tableAname, desc);
// enable the table
admin.enableTable(tableAname);
// Test that attribute changes were applied
desc = a.getTableDescriptor();
assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
// check HTD attribute
value = desc.getValue(attrName);
assertFalse("missing HTD attribute value", value == null);
assertFalse("HTD attribute value is incorrect",
Bytes.compareTo(value, attrValue) != 0);
// check HCD attribute
for (HColumnDescriptor c : desc.getFamilies()) {
value = c.getValue(attrName);
assertFalse("missing HCD attribute value", value == null);
assertFalse("HCD attribute value is incorrect",
Bytes.compareTo(value, attrValue) != 0);
}
}
@Test
public void testGetClosestRowBefore() throws IOException, InterruptedException {
final TableName tableName = TableName.valueOf(name.getMethodName());
final byte[] firstRow = Bytes.toBytes("row111");
final byte[] secondRow = Bytes.toBytes("row222");
final byte[] thirdRow = Bytes.toBytes("row333");
final byte[] forthRow = Bytes.toBytes("row444");
final byte[] beforeFirstRow = Bytes.toBytes("row");
final byte[] beforeSecondRow = Bytes.toBytes("row22");
final byte[] beforeThirdRow = Bytes.toBytes("row33");
final byte[] beforeForthRow = Bytes.toBytes("row44");
try (Table table =
TEST_UTIL.createTable(tableName,
new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1, 1024);
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
// set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow
// in Store.rowAtOrBeforeFromStoreFile
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
Region region =
TEST_UTIL.getRSForFirstRegionInTable(tableName).getFromOnlineRegions(regionName);
Put put1 = new Put(firstRow);
Put put2 = new Put(secondRow);
Put put3 = new Put(thirdRow);
Put put4 = new Put(forthRow);
byte[] one = new byte[] { 1 };
byte[] two = new byte[] { 2 };
byte[] three = new byte[] { 3 };
byte[] four = new byte[] { 4 };
put1.addColumn(HConstants.CATALOG_FAMILY, null, one);
put2.addColumn(HConstants.CATALOG_FAMILY, null, two);
put3.addColumn(HConstants.CATALOG_FAMILY, null, three);
put4.addColumn(HConstants.CATALOG_FAMILY, null, four);
table.put(put1);
table.put(put2);
table.put(put3);
table.put(put4);
region.flush(true);
Result result;
// Test before first that null is returned
result = getReverseScanResult(table, beforeFirstRow,
HConstants.CATALOG_FAMILY);
assertNull(result);
// Test at first that first is returned
result = getReverseScanResult(table, firstRow, HConstants.CATALOG_FAMILY);
assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
assertTrue(Bytes.equals(result.getRow(), firstRow));
assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
// Test in between first and second that first is returned
result = getReverseScanResult(table, beforeSecondRow, HConstants.CATALOG_FAMILY);
assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
assertTrue(Bytes.equals(result.getRow(), firstRow));
assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
// Test at second make sure second is returned
result = getReverseScanResult(table, secondRow, HConstants.CATALOG_FAMILY);
assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
assertTrue(Bytes.equals(result.getRow(), secondRow));
assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
// Test in second and third, make sure second is returned
result = getReverseScanResult(table, beforeThirdRow, HConstants.CATALOG_FAMILY);
assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
assertTrue(Bytes.equals(result.getRow(), secondRow));
assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
// Test at third make sure third is returned
result = getReverseScanResult(table, thirdRow, HConstants.CATALOG_FAMILY);
assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
assertTrue(Bytes.equals(result.getRow(), thirdRow));
assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
// Test in third and forth, make sure third is returned
result = getReverseScanResult(table, beforeForthRow, HConstants.CATALOG_FAMILY);
assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
assertTrue(Bytes.equals(result.getRow(), thirdRow));
assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
// Test at forth make sure forth is returned
result = getReverseScanResult(table, forthRow, HConstants.CATALOG_FAMILY);
assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
assertTrue(Bytes.equals(result.getRow(), forthRow));
assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
// Test after forth make sure forth is returned
result = getReverseScanResult(table, Bytes.add(forthRow, one), HConstants.CATALOG_FAMILY);
assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
assertTrue(Bytes.equals(result.getRow(), forthRow));
assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
}
}
private Result getReverseScanResult(Table table, byte[] row, byte[] fam) throws IOException {
Scan scan = new Scan(row);
scan.setSmall(true);
scan.setReversed(true);
scan.setCaching(1);
scan.addFamily(fam);
try (ResultScanner scanner = table.getScanner(scan)) {
return scanner.next();
}
}
/**
* For HBASE-2156
* @throws Exception
*/
@Test
public void testScanVariableReuse() throws Exception {
Scan scan = new Scan();
scan.addFamily(FAMILY);
scan.addColumn(FAMILY, ROW);
assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1);
scan = new Scan();
scan.addFamily(FAMILY);
assertTrue(scan.getFamilyMap().get(FAMILY) == null);
assertTrue(scan.getFamilyMap().containsKey(FAMILY));
}
@Test
public void testMultiRowMutation() throws Exception {
LOG.info("Starting testMultiRowMutation");
final TableName tableName = TableName.valueOf(name.getMethodName());
final byte [] ROW1 = Bytes.toBytes("testRow1");
Table t = TEST_UTIL.createTable(tableName, FAMILY);
Put p = new Put(ROW);
p.addColumn(FAMILY, QUALIFIER, VALUE);
MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, p);
p = new Put(ROW1);
p.addColumn(FAMILY, QUALIFIER, VALUE);
MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, p);
MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
mrmBuilder.addMutationRequest(m1);
mrmBuilder.addMutationRequest(m2);
MutateRowsRequest mrm = mrmBuilder.build();
CoprocessorRpcChannel channel = t.coprocessorService(ROW);
MultiRowMutationService.BlockingInterface service =
MultiRowMutationService.newBlockingStub(channel);
service.mutateRows(null, mrm);
Get g = new Get(ROW);
Result r = t.get(g);
assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
g = new Get(ROW1);
r = t.get(g);
assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
}
@Test
public void testRowMutation() throws Exception {
LOG.info("Starting testRowMutation");
final TableName tableName = TableName.valueOf(name.getMethodName());
Table t = TEST_UTIL.createTable(tableName, FAMILY);
byte [][] QUALIFIERS = new byte [][] {
Bytes.toBytes("a"), Bytes.toBytes("b")
};
RowMutations arm = new RowMutations(ROW);
Put p = new Put(ROW);
p.addColumn(FAMILY, QUALIFIERS[0], VALUE);
arm.add(p);
t.mutateRow(arm);
Get g = new Get(ROW);
Result r = t.get(g);
assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
arm = new RowMutations(ROW);
p = new Put(ROW);
p.addColumn(FAMILY, QUALIFIERS[1], VALUE);
arm.add(p);
Delete d = new Delete(ROW);
d.addColumns(FAMILY, QUALIFIERS[0]);
arm.add(d);
// TODO: Trying mutateRow again. The batch was failing with a one try only.
t.mutateRow(arm);
r = t.get(g);
assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
//Test that we get a region level exception
try {
arm = new RowMutations(ROW);
p = new Put(ROW);
p.addColumn(new byte[]{'b', 'o', 'g', 'u', 's'}, QUALIFIERS[0], VALUE);
arm.add(p);
t.mutateRow(arm);
fail("Expected NoSuchColumnFamilyException");
} catch(RetriesExhaustedWithDetailsException e) {
for(Throwable rootCause: e.getCauses()){
if(rootCause instanceof NoSuchColumnFamilyException){
return;
}
}
throw e;
}
}
@Test
public void testBatchAppendWithReturnResultFalse() throws Exception {
LOG.info("Starting testBatchAppendWithReturnResultFalse");
final TableName tableName = TableName.valueOf(name.getMethodName());
Table table = TEST_UTIL.createTable(tableName, FAMILY);
Append append1 = new Append(Bytes.toBytes("row1"));
append1.setReturnResults(false);
append1.add(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value1"));
Append append2 = new Append(Bytes.toBytes("row1"));
append2.setReturnResults(false);
append2.add(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value2"));
List<Append> appends = new ArrayList<>();
appends.add(append1);
appends.add(append2);
Object[] results = new Object[2];
table.batch(appends, results);
assertTrue(results.length == 2);
for(Object r : results) {
Result result = (Result)r;
assertTrue(result.isEmpty());
}
table.close();
}
@Test
public void testAppend() throws Exception {
LOG.info("Starting testAppend");
final TableName tableName = TableName.valueOf(name.getMethodName());
Table t = TEST_UTIL.createTable(tableName, FAMILY);
byte[] v1 = Bytes.toBytes("42");
byte[] v2 = Bytes.toBytes("23");
byte [][] QUALIFIERS = new byte [][] {
Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c")
};
Append a = new Append(ROW);
a.add(FAMILY, QUALIFIERS[0], v1);
a.add(FAMILY, QUALIFIERS[1], v2);
a.setReturnResults(false);
assertEmptyResult(t.append(a));
a = new Append(ROW);
a.add(FAMILY, QUALIFIERS[0], v2);
a.add(FAMILY, QUALIFIERS[1], v1);
a.add(FAMILY, QUALIFIERS[2], v2);
Result r = t.append(a);
assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0])));
assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1])));
// QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
assertEquals(r.getColumnLatestCell(FAMILY, QUALIFIERS[0]).getTimestamp(),
r.getColumnLatestCell(FAMILY, QUALIFIERS[2]).getTimestamp());
}
private List<Result> doAppend(final boolean walUsed) throws IOException {
LOG.info("Starting testAppend, walUsed is " + walUsed);
final TableName TABLENAME = TableName.valueOf(walUsed ? "testAppendWithWAL" : "testAppendWithoutWAL");
Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
final byte[] row1 = Bytes.toBytes("c");
final byte[] row2 = Bytes.toBytes("b");
final byte[] row3 = Bytes.toBytes("a");
final byte[] qual = Bytes.toBytes("qual");
Put put_0 = new Put(row2);
put_0.addColumn(FAMILY, qual, Bytes.toBytes("put"));
Put put_1 = new Put(row3);
put_1.addColumn(FAMILY, qual, Bytes.toBytes("put"));
Append append_0 = new Append(row1);
append_0.add(FAMILY, qual, Bytes.toBytes("i"));
Append append_1 = new Append(row1);
append_1.add(FAMILY, qual, Bytes.toBytes("k"));
Append append_2 = new Append(row1);
append_2.add(FAMILY, qual, Bytes.toBytes("e"));
if (!walUsed) {
append_2.setDurability(Durability.SKIP_WAL);
}
Append append_3 = new Append(row1);
append_3.add(FAMILY, qual, Bytes.toBytes("a"));
Scan s = new Scan();
s.setCaching(1);
t.append(append_0);
t.put(put_0);
t.put(put_1);
List<Result> results = new LinkedList<>();
try (ResultScanner scanner = t.getScanner(s)) {
t.append(append_1);
t.append(append_2);
t.append(append_3);
for (Result r : scanner) {
results.add(r);
}
}
TEST_UTIL.deleteTable(TABLENAME);
return results;
}
@Test
public void testAppendWithoutWAL() throws Exception {
List<Result> resultsWithWal = doAppend(true);
List<Result> resultsWithoutWal = doAppend(false);
assertEquals(resultsWithWal.size(), resultsWithoutWal.size());
for (int i = 0; i != resultsWithWal.size(); ++i) {
Result resultWithWal = resultsWithWal.get(i);
Result resultWithoutWal = resultsWithoutWal.get(i);
assertEquals(resultWithWal.rawCells().length, resultWithoutWal.rawCells().length);
for (int j = 0; j != resultWithWal.rawCells().length; ++j) {
Cell cellWithWal = resultWithWal.rawCells()[j];
Cell cellWithoutWal = resultWithoutWal.rawCells()[j];
assertTrue(Bytes.equals(CellUtil.cloneRow(cellWithWal), CellUtil.cloneRow(cellWithoutWal)));
assertTrue(Bytes.equals(CellUtil.cloneFamily(cellWithWal), CellUtil.cloneFamily(cellWithoutWal)));
assertTrue(Bytes.equals(CellUtil.cloneQualifier(cellWithWal), CellUtil.cloneQualifier(cellWithoutWal)));
assertTrue(Bytes.equals(CellUtil.cloneValue(cellWithWal), CellUtil.cloneValue(cellWithoutWal)));
}
}
}
@Test
public void testClientPoolRoundRobin() throws IOException {
final TableName tableName = TableName.valueOf(name.getMethodName());
int poolSize = 3;
int numVersions = poolSize * 2;
Configuration conf = TEST_UTIL.getConfiguration();
conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, Integer.MAX_VALUE);
final long ts = EnvironmentEdgeManager.currentTime();
Get get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions();
for (int versions = 1; versions <= numVersions; versions++) {
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
table.put(put);
Result result = table.get(get);
NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
.get(QUALIFIER);
assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER
+ " did not match " + versions, versions, navigableMap.size());
for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
assertTrue("The value at time " + entry.getKey()
+ " did not match what was put",
Bytes.equals(VALUE, entry.getValue()));
}
}
}
@Ignore ("Flakey: HBASE-8989") @Test
public void testClientPoolThreadLocal() throws IOException {
final TableName tableName = TableName.valueOf(name.getMethodName());
int poolSize = Integer.MAX_VALUE;
int numVersions = 3;
Configuration conf = TEST_UTIL.getConfiguration();
conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
final Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 3);
final long ts = EnvironmentEdgeManager.currentTime();
final Get get = new Get(ROW);
get.addColumn(FAMILY, QUALIFIER);
get.setMaxVersions();
for (int versions = 1; versions <= numVersions; versions++) {
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
table.put(put);
Result result = table.get(get);
NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
.get(QUALIFIER);
assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER + " did not match " +
versions + "; " + put.toString() + ", " + get.toString(), versions, navigableMap.size());
for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
assertTrue("The value at time " + entry.getKey()
+ " did not match what was put",
Bytes.equals(VALUE, entry.getValue()));
}
}
final Object waitLock = new Object();
ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
final AtomicReference<AssertionError> error = new AtomicReference<>(null);
for (int versions = numVersions; versions < numVersions * 2; versions++) {
final int versionsCopy = versions;
executorService.submit(new Callable<Void>() {
@Override
public Void call() {
try {
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
table.put(put);
Result result = table.get(get);
NavigableMap<Long, byte[]> navigableMap = result.getMap()
.get(FAMILY).get(QUALIFIER);
assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
+ Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy,
navigableMap.size());
for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
assertTrue("The value at time " + entry.getKey()
+ " did not match what was put",
Bytes.equals(VALUE, entry.getValue()));
}
synchronized (waitLock) {
waitLock.wait();
}
} catch (Exception e) {
} catch (AssertionError e) {
// the error happens in a thread, it won't fail the test,
// need to pass it to the caller for proper handling.
error.set(e);
LOG.error(e);
}
return null;
}
});
}
synchronized (waitLock) {
waitLock.notifyAll();
}
executorService.shutdownNow();
assertNull(error.get());
}
@Test
public void testCheckAndPut() throws IOException {
final byte [] anotherrow = Bytes.toBytes("anotherrow");
final byte [] value2 = Bytes.toBytes("abcd");
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
Put put1 = new Put(ROW);
put1.addColumn(FAMILY, QUALIFIER, VALUE);
// row doesn't exist, so using non-null value should be considered "not match".
boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put1);
assertEquals(ok, false);
// row doesn't exist, so using "null" to check for existence should be considered "match".
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
assertEquals(ok, true);
// row now exists, so using "null" to check for existence should be considered "not match".
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
assertEquals(ok, false);
Put put2 = new Put(ROW);
put2.addColumn(FAMILY, QUALIFIER, value2);
// row now exists, use the matching value to check
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put2);
assertEquals(ok, true);
Put put3 = new Put(anotherrow);
put3.addColumn(FAMILY, QUALIFIER, VALUE);
// try to do CheckAndPut on different rows
try {
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, value2, put3);
fail("trying to check and modify different rows should have failed.");
} catch(Exception e) {}
}
@Test
public void testCheckAndPutWithCompareOp() throws IOException {
final byte [] value1 = Bytes.toBytes("aaaa");
final byte [] value2 = Bytes.toBytes("bbbb");
final byte [] value3 = Bytes.toBytes("cccc");
final byte [] value4 = Bytes.toBytes("dddd");
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
Put put2 = new Put(ROW);
put2.addColumn(FAMILY, QUALIFIER, value2);
Put put3 = new Put(ROW);
put3.addColumn(FAMILY, QUALIFIER, value3);
// row doesn't exist, so using "null" to check for existence should be considered "match".
boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put2);
assertEquals(ok, true);
// cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
// turns out "match"
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value1, put2);
assertEquals(ok, false);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1, put2);
assertEquals(ok, false);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value1, put2);
assertEquals(ok, false);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1, put2);
assertEquals(ok, true);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value1, put2);
assertEquals(ok, true);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value1, put3);
assertEquals(ok, true);
// cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
// turns out "match"
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4, put3);
assertEquals(ok, false);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value4, put3);
assertEquals(ok, false);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4, put3);
assertEquals(ok, false);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value4, put3);
assertEquals(ok, true);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value4, put3);
assertEquals(ok, true);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value4, put2);
assertEquals(ok, true);
// cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
// turns out "match"
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value2, put2);
assertEquals(ok, false);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value2, put2);
assertEquals(ok, false);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value2, put2);
assertEquals(ok, false);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value2, put2);
assertEquals(ok, true);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value2, put2);
assertEquals(ok, true);
ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value2, put3);
assertEquals(ok, true);
}
@Test
public void testCheckAndDelete() throws IOException {
final byte [] value1 = Bytes.toBytes("aaaa");
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
FAMILY);
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, value1);
table.put(put);
Delete delete = new Delete(ROW);
delete.addColumns(FAMILY, QUALIFIER);
boolean ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, value1, delete);
assertEquals(ok, true);
}
@Test
public void testCheckAndDeleteWithCompareOp() throws IOException {
final byte [] value1 = Bytes.toBytes("aaaa");
final byte [] value2 = Bytes.toBytes("bbbb");
final byte [] value3 = Bytes.toBytes("cccc");
final byte [] value4 = Bytes.toBytes("dddd");
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
FAMILY);
Put put2 = new Put(ROW);
put2.addColumn(FAMILY, QUALIFIER, value2);
table.put(put2);
Put put3 = new Put(ROW);
put3.addColumn(FAMILY, QUALIFIER, value3);
Delete delete = new Delete(ROW);
delete.addColumns(FAMILY, QUALIFIER);
// cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
// turns out "match"
boolean ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value1, delete);
assertEquals(ok, false);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1, delete);
assertEquals(ok, false);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value1, delete);
assertEquals(ok, false);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1, delete);
assertEquals(ok, true);
table.put(put2);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value1, delete);
assertEquals(ok, true);
table.put(put2);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value1, delete);
assertEquals(ok, true);
// cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
// turns out "match"
table.put(put3);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4, delete);
assertEquals(ok, false);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value4, delete);
assertEquals(ok, false);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4, delete);
assertEquals(ok, false);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value4, delete);
assertEquals(ok, true);
table.put(put3);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value4, delete);
assertEquals(ok, true);
table.put(put3);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value4, delete);
assertEquals(ok, true);
// cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
// turns out "match"
table.put(put2);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value2, delete);
assertEquals(ok, false);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value2, delete);
assertEquals(ok, false);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value2, delete);
assertEquals(ok, false);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value2, delete);
assertEquals(ok, true);
table.put(put2);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value2, delete);
assertEquals(ok, true);
table.put(put2);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value2, delete);
assertEquals(ok, true);
}
/**
* Test ScanMetrics
* @throws Exception
*/
@Test
@SuppressWarnings ("unused")
public void testScanMetrics() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
// Set up test table:
// Create table:
Table ht = TEST_UTIL.createMultiRegionTable(tableName, FAMILY);
int numOfRegions = -1;
try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
numOfRegions = r.getStartKeys().length;
}
// Create 3 rows in the table, with rowkeys starting with "zzz*" so that
// scan are forced to hit all the regions.
Put put1 = new Put(Bytes.toBytes("zzz1"));
put1.addColumn(FAMILY, QUALIFIER, VALUE);
Put put2 = new Put(Bytes.toBytes("zzz2"));
put2.addColumn(FAMILY, QUALIFIER, VALUE);
Put put3 = new Put(Bytes.toBytes("zzz3"));
put3.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(Arrays.asList(put1, put2, put3));
Scan scan1 = new Scan();
int numRecords = 0;
ResultScanner scanner = ht.getScanner(scan1);
for(Result result : scanner) {
numRecords++;
}
scanner.close();
LOG.info("test data has " + numRecords + " records.");
// by default, scan metrics collection is turned off
assertEquals(null, scan1.getScanMetrics());
// turn on scan metrics
Scan scan2 = new Scan();
scan2.setScanMetricsEnabled(true);
scan2.setCaching(numRecords+1);
scanner = ht.getScanner(scan2);
for (Result result : scanner.next(numRecords - 1)) {
}
scanner.close();
// closing the scanner will set the metrics.
assertNotNull(scan2.getScanMetrics());
// set caching to 1, because metrics are collected in each roundtrip only
scan2 = new Scan();
scan2.setScanMetricsEnabled(true);
scan2.setCaching(1);
scanner = ht.getScanner(scan2);
// per HBASE-5717, this should still collect even if you don't run all the way to
// the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
for (Result result : scanner.next(numRecords - 1)) {
}
scanner.close();
ScanMetrics scanMetrics = scan2.getScanMetrics();
assertEquals("Did not access all the regions in the table", numOfRegions,
scanMetrics.countOfRegions.get());
// check byte counters
scan2 = new Scan();
scan2.setScanMetricsEnabled(true);
scan2.setCaching(1);
scanner = ht.getScanner(scan2);
int numBytes = 0;
for (Result result : scanner.next(1)) {
for (Cell cell: result.listCells()) {
numBytes += CellUtil.estimatedSerializedSizeOf(cell);
}
}
scanner.close();
scanMetrics = scan2.getScanMetrics();
assertEquals("Did not count the result bytes", numBytes,
scanMetrics.countOfBytesInResults.get());
// check byte counters on a small scan
scan2 = new Scan();
scan2.setScanMetricsEnabled(true);
scan2.setCaching(1);
scan2.setSmall(true);
scanner = ht.getScanner(scan2);
numBytes = 0;
for (Result result : scanner.next(1)) {
for (Cell cell: result.listCells()) {
numBytes += CellUtil.estimatedSerializedSizeOf(cell);
}
}
scanner.close();
scanMetrics = scan2.getScanMetrics();
assertEquals("Did not count the result bytes", numBytes,
scanMetrics.countOfBytesInResults.get());
// now, test that the metrics are still collected even if you don't call close, but do
// run past the end of all the records
/** There seems to be a timing issue here. Comment out for now. Fix when time.
Scan scanWithoutClose = new Scan();
scanWithoutClose.setCaching(1);
scanWithoutClose.setScanMetricsEnabled(true);
ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
for (Result result : scannerWithoutClose.next(numRecords + 1)) {
}
ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
assertEquals("Did not access all the regions in the table", numOfRegions,
scanMetricsWithoutClose.countOfRegions.get());
*/
// finally, test that the metrics are collected correctly if you both run past all the records,
// AND close the scanner
Scan scanWithClose = new Scan();
// make sure we can set caching up to the number of a scanned values
scanWithClose.setCaching(numRecords);
scanWithClose.setScanMetricsEnabled(true);
ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
for (Result result : scannerWithClose.next(numRecords + 1)) {
}
scannerWithClose.close();
ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose);
assertEquals("Did not access all the regions in the table", numOfRegions,
scanMetricsWithClose.countOfRegions.get());
}
private ScanMetrics getScanMetrics(Scan scan) throws Exception {
byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
assertTrue("Serialized metrics were not found.", serializedMetrics != null);
ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics);
return scanMetrics;
}
/**
* Tests that cache on write works all the way up from the client-side.
*
* Performs inserts, flushes, and compactions, verifying changes in the block
* cache along the way.
*
* @throws Exception
*/
@Test
public void testCacheOnWriteEvictOnClose() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [] data = Bytes.toBytes("data");
Table table = TEST_UTIL.createTable(tableName, FAMILY);
try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
// get the block cache and region
String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName)
.getFromOnlineRegions(regionName);
Store store = region.getStores().iterator().next();
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
BlockCache cache = cacheConf.getBlockCache();
// establish baseline stats
long startBlockCount = cache.getBlockCount();
long startBlockHits = cache.getStats().getHitCount();
long startBlockMiss = cache.getStats().getMissCount();
// wait till baseline is stable, (minimal 500 ms)
for (int i = 0; i < 5; i++) {
Thread.sleep(100);
if (startBlockCount != cache.getBlockCount()
|| startBlockHits != cache.getStats().getHitCount()
|| startBlockMiss != cache.getStats().getMissCount()) {
startBlockCount = cache.getBlockCount();
startBlockHits = cache.getStats().getHitCount();
startBlockMiss = cache.getStats().getMissCount();
i = -1;
}
}
// insert data
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, data);
table.put(put);
assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
// data was in memstore so don't expect any changes
assertEquals(startBlockCount, cache.getBlockCount());
assertEquals(startBlockHits, cache.getStats().getHitCount());
assertEquals(startBlockMiss, cache.getStats().getMissCount());
// flush the data
System.out.println("Flushing cache");
region.flush(true);
// expect one more block in cache, no change in hits/misses
long expectedBlockCount = startBlockCount + 1;
long expectedBlockHits = startBlockHits;
long expectedBlockMiss = startBlockMiss;
assertEquals(expectedBlockCount, cache.getBlockCount());
assertEquals(expectedBlockHits, cache.getStats().getHitCount());
assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
// read the data and expect same blocks, one new hit, no misses
assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
assertEquals(expectedBlockCount, cache.getBlockCount());
assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
// insert a second column, read the row, no new blocks, one new hit
byte [] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
byte [] data2 = Bytes.add(data, data);
put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER2, data2);
table.put(put);
Result r = table.get(new Get(ROW));
assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
assertEquals(expectedBlockCount, cache.getBlockCount());
assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
// flush, one new block
System.out.println("Flushing cache");
region.flush(true);
assertEquals(++expectedBlockCount, cache.getBlockCount());
assertEquals(expectedBlockHits, cache.getStats().getHitCount());
assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
// compact, net minus two blocks, two hits, no misses
System.out.println("Compacting");
assertEquals(2, store.getStorefilesCount());
store.triggerMajorCompaction();
region.compact(true);
store.closeAndArchiveCompactedFiles();
waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
assertEquals(1, store.getStorefilesCount());
expectedBlockCount -= 2; // evicted two blocks, cached none
assertEquals(expectedBlockCount, cache.getBlockCount());
expectedBlockHits += 2;
assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
assertEquals(expectedBlockHits, cache.getStats().getHitCount());
// read the row, this should be a cache miss because we don't cache data
// blocks on compaction
r = table.get(new Get(ROW));
assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
expectedBlockCount += 1; // cached one data block
assertEquals(expectedBlockCount, cache.getBlockCount());
assertEquals(expectedBlockHits, cache.getStats().getHitCount());
assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
}
}
private void waitForStoreFileCount(Store store, int count, int timeout)
throws InterruptedException {
long start = System.currentTimeMillis();
while (start + timeout > System.currentTimeMillis() &&
store.getStorefilesCount() != count) {
Thread.sleep(100);
}
System.out.println("start=" + start + ", now=" +
System.currentTimeMillis() + ", cur=" + store.getStorefilesCount());
assertEquals(count, store.getStorefilesCount());
}
@Test
/**
* Tests the non cached version of getRegionLocator by moving a region.
*/
public void testNonCachedGetRegionLocation() throws Exception {
// Test Initialization.
final TableName tableName = TableName.valueOf(name.getMethodName());
byte [] family1 = Bytes.toBytes("f1");
byte [] family2 = Bytes.toBytes("f2");
try (Table table = TEST_UTIL.createTable(tableName, new byte[][] {family1, family2}, 10);
Admin admin = TEST_UTIL.getAdmin();
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations();
assertEquals(1, allRegionLocations.size());
HRegionInfo regionInfo = allRegionLocations.get(0).getRegionInfo();
ServerName addrBefore = allRegionLocations.get(0).getServerName();
// Verify region location before move.
HRegionLocation addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
HRegionLocation addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true);
assertEquals(addrBefore.getPort(), addrCache.getPort());
assertEquals(addrBefore.getPort(), addrNoCache.getPort());
ServerName addrAfter = null;
// Now move the region to a different server.
for (int i = 0; i < SLAVES; i++) {
HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
ServerName addr = regionServer.getServerName();
if (addr.getPort() != addrBefore.getPort()) {
admin.move(regionInfo.getEncodedNameAsBytes(),
Bytes.toBytes(addr.toString()));
// Wait for the region to move.
Thread.sleep(5000);
addrAfter = addr;
break;
}
}
// Verify the region was moved.
addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true);
assertNotNull(addrAfter);
assertTrue(addrAfter.getPort() != addrCache.getPort());
assertEquals(addrAfter.getPort(), addrNoCache.getPort());
}
}
@Test
/**
* Tests getRegionsInRange by creating some regions over which a range of
* keys spans; then changing the key range.
*/
public void testGetRegionsInRange() throws Exception {
// Test Initialization.
byte [] startKey = Bytes.toBytes("ddc");
byte [] endKey = Bytes.toBytes("mmm");
final TableName tableName = TableName.valueOf(name.getMethodName());
Table t = TEST_UTIL.createMultiRegionTable(tableName, new byte[][] { FAMILY }, 10);
int numOfRegions = -1;
try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
numOfRegions = r.getStartKeys().length;
}
assertEquals(26, numOfRegions);
// Get the regions in this range
List<HRegionLocation> regionsList = getRegionsInRange(tableName, startKey, endKey);
assertEquals(10, regionsList.size());
// Change the start key
startKey = Bytes.toBytes("fff");
regionsList = getRegionsInRange(tableName, startKey, endKey);
assertEquals(7, regionsList.size());
// Change the end key
endKey = Bytes.toBytes("nnn");
regionsList = getRegionsInRange(tableName, startKey, endKey);
assertEquals(8, regionsList.size());
// Empty start key
regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW, endKey);
assertEquals(13, regionsList.size());
// Empty end key
regionsList = getRegionsInRange(tableName, startKey, HConstants.EMPTY_END_ROW);
assertEquals(21, regionsList.size());
// Both start and end keys empty
regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW,
HConstants.EMPTY_END_ROW);
assertEquals(26, regionsList.size());
// Change the end key to somewhere in the last block
endKey = Bytes.toBytes("zzz1");
regionsList = getRegionsInRange(tableName, startKey, endKey);
assertEquals(21, regionsList.size());
// Change the start key to somewhere in the first block
startKey = Bytes.toBytes("aac");
regionsList = getRegionsInRange(tableName, startKey, endKey);
assertEquals(26, regionsList.size());
// Make start and end key the same
startKey = endKey = Bytes.toBytes("ccc");
regionsList = getRegionsInRange(tableName, startKey, endKey);
assertEquals(1, regionsList.size());
}
private List<HRegionLocation> getRegionsInRange(TableName tableName, byte[] startKey,
byte[] endKey) throws IOException {
List<HRegionLocation> regionsInRange = new ArrayList<>();
byte[] currentKey = startKey;
final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW);
try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName);) {
do {
HRegionLocation regionLocation = r.getRegionLocation(currentKey);
regionsInRange.add(regionLocation);
currentKey = regionLocation.getRegionInfo().getEndKey();
} while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)
&& (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0));
return regionsInRange;
}
}
@Test
public void testJira6912() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table foo = TEST_UTIL.createTable(tableName, new byte[][] {FAMILY}, 10);
List<Put> puts = new ArrayList<Put>();
for (int i=0;i !=100; i++){
Put put = new Put(Bytes.toBytes(i));
put.addColumn(FAMILY, FAMILY, Bytes.toBytes(i));
puts.add(put);
}
foo.put(puts);
// If i comment this out it works
TEST_UTIL.flush();
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(1));
scan.setStopRow(Bytes.toBytes(3));
scan.addColumn(FAMILY, FAMILY);
scan.setFilter(new RowFilter(CompareFilter.CompareOp.NOT_EQUAL,
new BinaryComparator(Bytes.toBytes(1))));
ResultScanner scanner = foo.getScanner(scan);
Result[] bar = scanner.next(100);
assertEquals(1, bar.length);
}
@Test
public void testScan_NullQualifier() throws IOException {
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, VALUE);
table.put(put);
put = new Put(ROW);
put.addColumn(FAMILY, null, VALUE);
table.put(put);
LOG.info("Row put");
Scan scan = new Scan();
scan.addColumn(FAMILY, null);
ResultScanner scanner = table.getScanner(scan);
Result[] bar = scanner.next(100);
assertEquals(1, bar.length);
assertEquals(1, bar[0].size());
scan = new Scan();
scan.addFamily(FAMILY);
scanner = table.getScanner(scan);
bar = scanner.next(100);
assertEquals(1, bar.length);
assertEquals(2, bar[0].size());
}
@Test
public void testNegativeTimestamp() throws IOException {
Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
try {
Put put = new Put(ROW, -1);
put.addColumn(FAMILY, QUALIFIER, VALUE);
table.put(put);
fail("Negative timestamps should not have been allowed");
} catch (IllegalArgumentException ex) {
assertTrue(ex.getMessage().contains("negative"));
}
try {
Put put = new Put(ROW);
long ts = -1;
put.addColumn(FAMILY, QUALIFIER, ts, VALUE);
table.put(put);
fail("Negative timestamps should not have been allowed");
} catch (IllegalArgumentException ex) {
assertTrue(ex.getMessage().contains("negative"));
}
try {
Delete delete = new Delete(ROW, -1);
table.delete(delete);
fail("Negative timestamps should not have been allowed");
} catch (IllegalArgumentException ex) {
assertTrue(ex.getMessage().contains("negative"));
}
try {
Delete delete = new Delete(ROW);
delete.addFamily(FAMILY, -1);
table.delete(delete);
fail("Negative timestamps should not have been allowed");
} catch (IllegalArgumentException ex) {
assertTrue(ex.getMessage().contains("negative"));
}
try {
Scan scan = new Scan();
scan.setTimeRange(-1, 1);
table.getScanner(scan);
fail("Negative timestamps should not have been allowed");
} catch (IllegalArgumentException ex) {
assertTrue(ex.getMessage().contains("negative"));
}
// KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user
// already has negative timestamps in cluster data, HBase won't be able to handle that
try {
new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1, Bytes.toBytes(42));
} catch (IllegalArgumentException ex) {
fail("KeyValue SHOULD allow negative timestamps");
}
table.close();
}
@Test
public void testRawScanRespectsVersions() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table table = TEST_UTIL.createTable(tableName, FAMILY);
byte[] row = Bytes.toBytes("row");
// put the same row 4 times, with different values
Put p = new Put(row);
p.addColumn(FAMILY, QUALIFIER, (long) 10, VALUE);
table.put(p);
p = new Put(row);
p.addColumn(FAMILY, QUALIFIER, (long) 11, ArrayUtils.add(VALUE, (byte) 2));
table.put(p);
p = new Put(row);
p.addColumn(FAMILY, QUALIFIER, (long) 12, ArrayUtils.add(VALUE, (byte) 3));
table.put(p);
p = new Put(row);
p.addColumn(FAMILY, QUALIFIER, (long) 13, ArrayUtils.add(VALUE, (byte) 4));
table.put(p);
int versions = 4;
Scan s = new Scan(row);
// get all the possible versions
s.setMaxVersions();
s.setRaw(true);
ResultScanner scanner = table.getScanner(s);
int count = 0;
for (Result r : scanner) {
assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
count++;
}
assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
count);
scanner.close();
// then if we decrease the number of versions, but keep the scan raw, we should see exactly that
// number of versions
versions = 2;
s.setMaxVersions(versions);
scanner = table.getScanner(s);
count = 0;
for (Result r : scanner) {
assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
count++;
}
assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
count);
scanner.close();
// finally, if we turn off raw scanning, but max out the number of versions, we should go back
// to seeing just three
versions = 3;
s.setMaxVersions(versions);
scanner = table.getScanner(s);
count = 0;
for (Result r : scanner) {
assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
count++;
}
assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
count);
scanner.close();
table.close();
TEST_UTIL.deleteTable(tableName);
}
@Test
public void testEmptyFilterList() throws Exception {
// Test Initialization.
final TableName tableName = TableName.valueOf(name.getMethodName());
Table table = TEST_UTIL.createTable(tableName, FAMILY);
// Insert one row each region
Put put = new Put(Bytes.toBytes("row"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
table.put(put);
List<Result> scanResults = new LinkedList<>();
Scan scan = new Scan();
scan.setFilter(new FilterList());
try (ResultScanner scanner = table.getScanner(scan)) {
for (Result r : scanner) {
scanResults.add(r);
}
}
assertEquals(1, scanResults.size());
Get g = new Get(Bytes.toBytes("row"));
g.setFilter(new FilterList());
Result getResult = table.get(g);
Result scanResult = scanResults.get(0);
assertEquals(scanResult.rawCells().length, getResult.rawCells().length);
for (int i = 0; i != scanResult.rawCells().length; ++i) {
Cell scanCell = scanResult.rawCells()[i];
Cell getCell = getResult.rawCells()[i];
assertEquals(0, Bytes.compareTo(CellUtil.cloneRow(scanCell), CellUtil.cloneRow(getCell)));
assertEquals(0, Bytes.compareTo(CellUtil.cloneFamily(scanCell), CellUtil.cloneFamily(getCell)));
assertEquals(0, Bytes.compareTo(CellUtil.cloneQualifier(scanCell), CellUtil.cloneQualifier(getCell)));
assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(scanCell), CellUtil.cloneValue(getCell)));
}
}
@Test
public void testSmallScan() throws Exception {
// Test Initialization.
final TableName tableName = TableName.valueOf(name.getMethodName());
Table table = TEST_UTIL.createTable(tableName, FAMILY);
// Insert one row each region
int insertNum = 10;
for (int i = 0; i < 10; i++) {
Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
put.addColumn(FAMILY, QUALIFIER, VALUE);
table.put(put);
}
// normal scan
ResultScanner scanner = table.getScanner(new Scan());
int count = 0;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
}
assertEquals(insertNum, count);
// small scan
Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
scan.setSmall(true);
scan.setCaching(2);
scanner = table.getScanner(scan);
count = 0;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
}
assertEquals(insertNum, count);
}
@Test
public void testSuperSimpleWithReverseScan() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
put = new Put(Bytes.toBytes("0-b11111-0000000000000000002"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
put = new Put(Bytes.toBytes("0-b11111-0000000000000000004"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
put = new Put(Bytes.toBytes("0-b11111-0000000000000000006"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
put = new Put(Bytes.toBytes("0-b11111-0000000000000000008"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
put = new Put(Bytes.toBytes("0-b22222-0000000000000000001"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
put = new Put(Bytes.toBytes("0-b22222-0000000000000000003"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
put = new Put(Bytes.toBytes("0-b22222-0000000000000000005"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
put = new Put(Bytes.toBytes("0-b22222-0000000000000000007"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
put = new Put(Bytes.toBytes("0-b22222-0000000000000000009"));
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
Scan scan = new Scan(Bytes.toBytes("0-b11111-9223372036854775807"),
Bytes.toBytes("0-b11111-0000000000000000000"));
scan.setReversed(true);
ResultScanner scanner = ht.getScanner(scan);
Result result = scanner.next();
assertTrue(Bytes.equals(result.getRow(),
Bytes.toBytes("0-b11111-0000000000000000008")));
scanner.close();
ht.close();
}
@Test
public void testFiltersWithReverseScan() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
byte[][] ROWS = makeN(ROW, 10);
byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
Bytes.toBytes("col1-<d2v1>-<d3v2>"),
Bytes.toBytes("col2-<d2v1>-<d3v2>"),
Bytes.toBytes("col3-<d2v1>-<d3v2>"),
Bytes.toBytes("col4-<d2v1>-<d3v2>"),
Bytes.toBytes("col5-<d2v1>-<d3v2>"),
Bytes.toBytes("col6-<d2v1>-<d3v2>"),
Bytes.toBytes("col7-<d2v1>-<d3v2>"),
Bytes.toBytes("col8-<d2v1>-<d3v2>"),
Bytes.toBytes("col9-<d2v1>-<d3v2>") };
for (int i = 0; i < 10; i++) {
Put put = new Put(ROWS[i]);
put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
ht.put(put);
}
Scan scan = new Scan();
scan.setReversed(true);
scan.addFamily(FAMILY);
Filter filter = new QualifierFilter(CompareOp.EQUAL,
new RegexStringComparator("col[1-5]"));
scan.setFilter(filter);
ResultScanner scanner = ht.getScanner(scan);
int expectedIndex = 5;
for (Result result : scanner) {
assertEquals(result.size(), 1);
Cell c = result.rawCells()[0];
assertTrue(Bytes.equals(c.getRowArray(), c.getRowOffset(), c.getRowLength(),
ROWS[expectedIndex], 0, ROWS[expectedIndex].length));
assertTrue(Bytes.equals(c.getQualifierArray(), c.getQualifierOffset(),
c.getQualifierLength(), QUALIFIERS[expectedIndex], 0, QUALIFIERS[expectedIndex].length));
expectedIndex--;
}
assertEquals(expectedIndex, 0);
scanner.close();
ht.close();
}
@Test
public void testKeyOnlyFilterWithReverseScan() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
byte[][] ROWS = makeN(ROW, 10);
byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
Bytes.toBytes("col1-<d2v1>-<d3v2>"),
Bytes.toBytes("col2-<d2v1>-<d3v2>"),
Bytes.toBytes("col3-<d2v1>-<d3v2>"),
Bytes.toBytes("col4-<d2v1>-<d3v2>"),
Bytes.toBytes("col5-<d2v1>-<d3v2>"),
Bytes.toBytes("col6-<d2v1>-<d3v2>"),
Bytes.toBytes("col7-<d2v1>-<d3v2>"),
Bytes.toBytes("col8-<d2v1>-<d3v2>"),
Bytes.toBytes("col9-<d2v1>-<d3v2>") };
for (int i = 0; i < 10; i++) {
Put put = new Put(ROWS[i]);
put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
ht.put(put);
}
Scan scan = new Scan();
scan.setReversed(true);
scan.addFamily(FAMILY);
Filter filter = new KeyOnlyFilter(true);
scan.setFilter(filter);
ResultScanner scanner = ht.getScanner(scan);
int count = 0;
for (Result result : ht.getScanner(scan)) {
assertEquals(result.size(), 1);
assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT);
assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length);
count++;
}
assertEquals(count, 10);
scanner.close();
ht.close();
}
/**
* Test simple table and non-existent row cases.
*/
@Test
public void testSimpleMissingWithReverseScan() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
byte[][] ROWS = makeN(ROW, 4);
// Try to get a row on an empty table
Scan scan = new Scan();
scan.setReversed(true);
Result result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan(ROWS[0]);
scan.setReversed(true);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan(ROWS[0], ROWS[1]);
scan.setReversed(true);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan();
scan.setReversed(true);
scan.addFamily(FAMILY);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
scan = new Scan();
scan.setReversed(true);
scan.addColumn(FAMILY, QUALIFIER);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
// Insert a row
Put put = new Put(ROWS[2]);
put.addColumn(FAMILY, QUALIFIER, VALUE);
ht.put(put);
// Make sure we can scan the row
scan = new Scan();
scan.setReversed(true);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
scan = new Scan(ROWS[3], ROWS[0]);
scan.setReversed(true);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
scan = new Scan(ROWS[2], ROWS[1]);
scan.setReversed(true);
result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
// Try to scan empty rows around it
// Introduced MemStore#shouldSeekForReverseScan to fix the following
scan = new Scan(ROWS[1]);
scan.setReversed(true);
result = getSingleScanResult(ht, scan);
assertNullResult(result);
ht.close();
}
@Test
public void testNullWithReverseScan() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
Table ht = TEST_UTIL.createTable(tableName, FAMILY);
// Null qualifier (should work)
Put put = new Put(ROW);
put.addColumn(FAMILY, null, VALUE);
ht.put(put);
scanTestNull(ht, ROW, FAMILY, VALUE, true);
Delete delete = new Delete(ROW);
delete.addColumns(FAMILY, null);
ht.delete(delete);
// Use a new table
ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName() + "2"), FAMILY);
// Empty qualifier, byte[0] instead of null (should work)
put = new Put(ROW);
put.addColumn(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
ht.put(put);
scanTestNull(ht, ROW, FAMILY, VALUE, true);
TEST_UTIL.flush();
scanTestNull(ht, ROW, FAMILY, VALUE, true);
delete = new Delete(ROW);
delete.addColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
ht.delete(delete);
// Null value
put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, null);
ht.put(put);
Scan scan = new Scan();
scan.setReversed(true);
scan.addColumn(FAMILY, QUALIFIER);
Result result = getSingleScanResult(ht, scan);
assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
ht.close();
}
@Test
public void testDeletesWithReverseScan() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
byte[][] ROWS = makeNAscii(ROW, 6);
byte[][] FAMILIES = makeNAscii(FAMILY, 3);
byte[][] VALUES = makeN(VALUE, 5);
long[] ts = { 1000, 2000, 3000, 4000, 5000 };
Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3);
Put put = new Put(ROW);
put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
put.addColumn(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
ht.put(put);
Delete delete = new Delete(ROW);
delete.addFamily(FAMILIES[0], ts[0]);
ht.delete(delete);
Scan scan = new Scan(ROW);
scan.setReversed(true);
scan.addFamily(FAMILIES[0]);
scan.setMaxVersions(Integer.MAX_VALUE);
Result result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1] },
new byte[][] { VALUES[1] }, 0, 0);
// Test delete latest version
put = new Put(ROW);
put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
put.addColumn(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
put.addColumn(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
put.addColumn(FAMILIES[0], null, ts[4], VALUES[4]);
put.addColumn(FAMILIES[0], null, ts[2], VALUES[2]);
put.addColumn(FAMILIES[0], null, ts[3], VALUES[3]);
ht.put(put);
delete = new Delete(ROW);
delete.addColumn(FAMILIES[0], QUALIFIER); // ts[4]
ht.delete(delete);
scan = new Scan(ROW);
scan.setReversed(true);
scan.addColumn(FAMILIES[0], QUALIFIER);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
// Test for HBASE-1847
delete = new Delete(ROW);
delete.addColumn(FAMILIES[0], null);
ht.delete(delete);
// Cleanup null qualifier
delete = new Delete(ROW);
delete.addColumns(FAMILIES[0], null);
ht.delete(delete);
// Expected client behavior might be that you can re-put deleted values
// But alas, this is not to be. We can't put them back in either case.
put = new Put(ROW);
put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
ht.put(put);
// The Scanner returns the previous values, the expected-naive-unexpected
// behavior
scan = new Scan(ROW);
scan.setReversed(true);
scan.addFamily(FAMILIES[0]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
// Test deleting an entire family from one row but not the other various
// ways
put = new Put(ROWS[0]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
ht.put(put);
put = new Put(ROWS[1]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
ht.put(put);
put = new Put(ROWS[2]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
ht.put(put);
delete = new Delete(ROWS[0]);
delete.addFamily(FAMILIES[2]);
ht.delete(delete);
delete = new Delete(ROWS[1]);
delete.addColumns(FAMILIES[1], QUALIFIER);
ht.delete(delete);
delete = new Delete(ROWS[2]);
delete.addColumn(FAMILIES[1], QUALIFIER);
delete.addColumn(FAMILIES[1], QUALIFIER);
delete.addColumn(FAMILIES[2], QUALIFIER);
ht.delete(delete);
scan = new Scan(ROWS[0]);
scan.setReversed(true);
scan.addFamily(FAMILIES[1]);
scan.addFamily(FAMILIES[2]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertTrue("Expected 2 keys but received " + result.size(),
result.size() == 2);
assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[] { ts[0],
ts[1] }, new byte[][] { VALUES[0], VALUES[1] }, 0, 1);
scan = new Scan(ROWS[1]);
scan.setReversed(true);
scan.addFamily(FAMILIES[1]);
scan.addFamily(FAMILIES[2]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertTrue("Expected 2 keys but received " + result.size(),
result.size() == 2);
scan = new Scan(ROWS[2]);
scan.setReversed(true);
scan.addFamily(FAMILIES[1]);
scan.addFamily(FAMILIES[2]);
scan.setMaxVersions(Integer.MAX_VALUE);
result = getSingleScanResult(ht, scan);
assertEquals(1, result.size());
assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
new long[] { ts[2] }, new byte[][] { VALUES[2] }, 0, 0);
// Test if we delete the family first in one row (HBASE-1541)
delete = new Delete(ROWS[3]);
delete.addFamily(FAMILIES[1]);
ht.delete(delete);
put = new Put(ROWS[3]);
put.addColumn(FAMILIES[2], QUALIFIER, VALUES[0]);
ht.put(put);
put = new Put(ROWS[4]);
put.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
put.addColumn(FAMILIES[2], QUALIFIER, VALUES[2]);
ht.put(put);
scan = new Scan(ROWS[4]);
scan.setReversed(true);
scan.addFamily(FAMILIES[1]);
scan.addFamily(FAMILIES[2]);
scan.setMaxVersions(Integer.MAX_VALUE);
ResultScanner scanner = ht.getScanner(scan);
result = scanner.next();
assertTrue("Expected 2 keys but received " + result.size(),
result.size() == 2);
assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
result = scanner.next();
assertTrue("Expected 1 key but received " + result.size(),
result.size() == 1);
assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
scanner.close();
ht.close();
}
/**
* Tests reversed scan under multi regions
*/
@Test
public void testReversedScanUnderMultiRegions() throws Exception {
// Test Initialization.
final TableName tableName = TableName.valueOf(name.getMethodName());
byte[] maxByteArray = ConnectionUtils.MAX_BYTE_ARRAY;
byte[][] splitRows = new byte[][] { Bytes.toBytes("005"),
Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)),
Bytes.toBytes("006"),
Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)),
Bytes.toBytes("007"),
Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)),
Bytes.toBytes("008"), Bytes.multiple(maxByteArray, 2) };
Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows);
TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
try(RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
assertEquals(splitRows.length + 1, l.getAllRegionLocations().size());
}
// Insert one row each region
int insertNum = splitRows.length;
for (int i = 0; i < insertNum; i++) {
Put put = new Put(splitRows[i]);
put.addColumn(FAMILY, QUALIFIER, VALUE);
table.put(put);
}
// scan forward
ResultScanner scanner = table.getScanner(new Scan());
int count = 0;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
}
assertEquals(insertNum, count);
// scan backward
Scan scan = new Scan();
scan.setReversed(true);
scanner = table.getScanner(scan);
count = 0;
byte[] lastRow = null;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
byte[] thisRow = r.getRow();
if (lastRow != null) {
assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
+ ",this row=" + Bytes.toString(thisRow),
Bytes.compareTo(thisRow, lastRow) < 0);
}
lastRow = thisRow;
}
assertEquals(insertNum, count);
table.close();
}
/**
* Tests reversed scan under multi regions
*/
@Test
public void testSmallReversedScanUnderMultiRegions() throws Exception {
// Test Initialization.
final TableName tableName = TableName.valueOf(name.getMethodName());
byte[][] splitRows = new byte[][]{
Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"),
Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010")};
Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows);
TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
assertEquals(splitRows.length + 1, l.getAllRegionLocations().size());
}
for (byte[] splitRow : splitRows) {
Put put = new Put(splitRow);
put.addColumn(FAMILY, QUALIFIER, VALUE);
table.put(put);
byte[] nextRow = Bytes.copy(splitRow);
nextRow[nextRow.length - 1]++;
put = new Put(nextRow);
put.addColumn(FAMILY, QUALIFIER, VALUE);
table.put(put);
}
// scan forward
ResultScanner scanner = table.getScanner(new Scan());
int count = 0;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
}
assertEquals(12, count);
reverseScanTest(table, false);
reverseScanTest(table, true);
table.close();
}
private void reverseScanTest(Table table, boolean small) throws IOException {
// scan backward
Scan scan = new Scan();
scan.setReversed(true);
ResultScanner scanner = table.getScanner(scan);
int count = 0;
byte[] lastRow = null;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
byte[] thisRow = r.getRow();
if (lastRow != null) {
assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
+ ",this row=" + Bytes.toString(thisRow),
Bytes.compareTo(thisRow, lastRow) < 0);
}
lastRow = thisRow;
}
assertEquals(12, count);
scan = new Scan();
scan.setSmall(small);
scan.setReversed(true);
scan.setStartRow(Bytes.toBytes("002"));
scanner = table.getScanner(scan);
count = 0;
lastRow = null;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
byte[] thisRow = r.getRow();
if (lastRow != null) {
assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
+ ",this row=" + Bytes.toString(thisRow),
Bytes.compareTo(thisRow, lastRow) < 0);
}
lastRow = thisRow;
}
assertEquals(3, count); // 000 001 002
scan = new Scan();
scan.setSmall(small);
scan.setReversed(true);
scan.setStartRow(Bytes.toBytes("002"));
scan.setStopRow(Bytes.toBytes("000"));
scanner = table.getScanner(scan);
count = 0;
lastRow = null;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
byte[] thisRow = r.getRow();
if (lastRow != null) {
assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
+ ",this row=" + Bytes.toString(thisRow),
Bytes.compareTo(thisRow, lastRow) < 0);
}
lastRow = thisRow;
}
assertEquals(2, count); // 001 002
scan = new Scan();
scan.setSmall(small);
scan.setReversed(true);
scan.setStartRow(Bytes.toBytes("001"));
scanner = table.getScanner(scan);
count = 0;
lastRow = null;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
byte[] thisRow = r.getRow();
if (lastRow != null) {
assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
+ ",this row=" + Bytes.toString(thisRow),
Bytes.compareTo(thisRow, lastRow) < 0);
}
lastRow = thisRow;
}
assertEquals(2, count); // 000 001
scan = new Scan();
scan.setSmall(small);
scan.setReversed(true);
scan.setStartRow(Bytes.toBytes("000"));
scanner = table.getScanner(scan);
count = 0;
lastRow = null;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
byte[] thisRow = r.getRow();
if (lastRow != null) {
assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
+ ",this row=" + Bytes.toString(thisRow),
Bytes.compareTo(thisRow, lastRow) < 0);
}
lastRow = thisRow;
}
assertEquals(1, count); // 000
scan = new Scan();
scan.setSmall(small);
scan.setReversed(true);
scan.setStartRow(Bytes.toBytes("006"));
scan.setStopRow(Bytes.toBytes("002"));
scanner = table.getScanner(scan);
count = 0;
lastRow = null;
for (Result r : scanner) {
assertTrue(!r.isEmpty());
count++;
byte[] thisRow = r.getRow();
if (lastRow != null) {
assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
+ ",this row=" + Bytes.toString(thisRow),
Bytes.compareTo(thisRow, lastRow) < 0);
}
lastRow = thisRow;
}
assertEquals(4, count); // 003 004 005 006
}
@Test
public void testGetStartEndKeysWithRegionReplicas() throws IOException {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
htd.addFamily(fam);
byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
Admin admin = TEST_UTIL.getAdmin();
admin.createTable(htd, KEYS);
List<HRegionInfo> regions = admin.getTableRegions(htd.getTableName());
HRegionLocator locator =
(HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
for (int regionReplication = 1; regionReplication < 4; regionReplication++) {
List<RegionLocations> regionLocations = new ArrayList<>();
// mock region locations coming from meta with multiple replicas
for (HRegionInfo region : regions) {
HRegionLocation[] arr = new HRegionLocation[regionReplication];
for (int i = 0; i < arr.length; i++) {
arr[i] = new HRegionLocation(RegionReplicaUtil.getRegionInfoForReplica(region, i), null);
}
regionLocations.add(new RegionLocations(arr));
}
Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys(regionLocations);
assertEquals(KEYS.length + 1, startEndKeys.getFirst().length);
for (int i = 0; i < KEYS.length + 1; i++) {
byte[] startKey = i == 0 ? HConstants.EMPTY_START_ROW : KEYS[i - 1];
byte[] endKey = i == KEYS.length ? HConstants.EMPTY_END_ROW : KEYS[i];
assertArrayEquals(startKey, startEndKeys.getFirst()[i]);
assertArrayEquals(endKey, startEndKeys.getSecond()[i]);
}
}
}
@Test
public void testFilterAllRecords() throws IOException {
Scan scan = new Scan();
scan.setBatch(1);
scan.setCaching(1);
// Filter out any records
scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0])));
try (Table table = TEST_UTIL.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME)) {
try (ResultScanner s = table.getScanner(scan)) {
assertNull(s.next());
}
}
}
@Test
public void testRegionCache() throws IOException {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
htd.addFamily(fam);
byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
Admin admin = TEST_UTIL.getAdmin();
admin.createTable(htd, KEYS);
HRegionLocator locator =
(HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
List<HRegionLocation> results = locator.getAllRegionLocations();
int number = ((ConnectionImplementation)admin.getConnection())
.getNumberOfCachedRegionLocations(htd.getTableName());
assertEquals(results.size(), number);
}
@Test
public void testCellSizeLimit() throws IOException {
final TableName tableName = TableName.valueOf("testCellSizeLimit");
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.setConfiguration(HRegion.HBASE_MAX_CELL_SIZE_KEY, Integer.toString(10 * 1024)); // 10K
HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
htd.addFamily(fam);
Admin admin = TEST_UTIL.getAdmin();
admin.createTable(htd);
// Will succeed
try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, Bytes.toBytes(0L)));
t.increment(new Increment(ROW).addColumn(FAMILY, QUALIFIER, 1L));
}
// Will succeed
try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[9*1024]));
}
// Will fail
try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
try {
t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[10 * 1024]));
fail("Oversize cell failed to trigger exception");
} catch (IOException e) {
// expected
}
try {
t.append(new Append(ROW).add(FAMILY, QUALIFIER, new byte[10 * 1024]));
fail("Oversize cell failed to trigger exception");
} catch (IOException e) {
// expected
}
}
}
}