/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. */ package org.apache.hadoop.hbase.client; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HTestConst; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; import org.apache.hadoop.hbase.filter.ColumnRangeFilter; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; /** * A client-side test, mostly testing scanners with various parameters. */ @Category(MediumTests.class) public class TestScannersFromClientSide { private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class); private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static byte [] ROW = Bytes.toBytes("testRow"); private static byte [] FAMILY = Bytes.toBytes("testFamily"); private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); private static byte [] VALUE = Bytes.toBytes("testValue"); /** * @throws java.lang.Exception */ @BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(3); } /** * @throws java.lang.Exception */ @AfterClass public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } /** * @throws java.lang.Exception */ @Before public void setUp() throws Exception { // Nothing to do. } /** * @throws java.lang.Exception */ @After public void tearDown() throws Exception { // Nothing to do. } /** * Test from client side for batch of scan * * @throws Exception */ @Test public void testScanBatch() throws Exception { byte [] TABLE = Bytes.toBytes("testScanBatch"); byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8); HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); Put put; Scan scan; Delete delete; Result result; ResultScanner scanner; boolean toLog = true; List<KeyValue> kvListExp; // table: row, family, c0:0, c1:1, ... , c7:7 put = new Put(ROW); for (int i=0; i < QUALIFIERS.length; i++) { KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE); put.add(kv); } ht.put(put); // table: row, family, c0:0, c1:1, ..., c6:2, c6:6 , c7:7 put = new Put(ROW); KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE); put.add(kv); ht.put(put); // delete upto ts: 3 delete = new Delete(ROW); delete.deleteFamily(FAMILY, 3); ht.delete(delete); // without batch scan = new Scan(ROW); scan.setMaxVersions(); scanner = ht.getScanner(scan); // c4:4, c5:5, c6:6, c7:7 kvListExp = new ArrayList<KeyValue>(); kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE)); result = scanner.next(); verifyResult(result, kvListExp, toLog, "Testing first batch of scan"); // with batch scan = new Scan(ROW); scan.setMaxVersions(); scan.setBatch(2); scanner = ht.getScanner(scan); // First batch: c4:4, c5:5 kvListExp = new ArrayList<KeyValue>(); kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE)); result = scanner.next(); verifyResult(result, kvListExp, toLog, "Testing first batch of scan"); // Second batch: c6:6, c7:7 kvListExp = new ArrayList<KeyValue>(); kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE)); result = scanner.next(); verifyResult(result, kvListExp, toLog, "Testing second batch of scan"); } /** * Test from client side for get with maxResultPerCF set * * @throws Exception */ @Test public void testGetMaxResults() throws Exception { byte [] TABLE = Bytes.toBytes("testGetMaxResults"); byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3); byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20); HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES); Get get; Put put; Result result; boolean toLog = true; List<KeyValue> kvListExp; kvListExp = new ArrayList<KeyValue>(); // Insert one CF for row[0] put = new Put(ROW); for (int i=0; i < 10; i++) { KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE); put.add(kv); kvListExp.add(kv); } ht.put(put); get = new Get(ROW); result = ht.get(get); verifyResult(result, kvListExp, toLog, "Testing without setting maxResults"); get = new Get(ROW); get.setMaxResultsPerColumnFamily(2); result = ht.get(get); kvListExp = new ArrayList<KeyValue>(); kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE)); verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults"); // Filters: ColumnRangeFilter get = new Get(ROW); get.setMaxResultsPerColumnFamily(5); get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5], true)); result = ht.get(get); kvListExp = new ArrayList<KeyValue>(); kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE)); verifyResult(result, kvListExp, toLog, "Testing single CF with CRF"); // Insert two more CF for row[0] // 20 columns for CF2, 10 columns for CF1 put = new Put(ROW); for (int i=0; i < QUALIFIERS.length; i++) { KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE); put.add(kv); } ht.put(put); put = new Put(ROW); for (int i=0; i < 10; i++) { KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE); put.add(kv); } ht.put(put); get = new Get(ROW); get.setMaxResultsPerColumnFamily(12); get.addFamily(FAMILIES[1]); get.addFamily(FAMILIES[2]); result = ht.get(get); kvListExp = new ArrayList<KeyValue>(); //Exp: CF1:q0, ..., q9, CF2: q0, q1, q10, q11, ..., q19 for (int i=0; i < 10; i++) { kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE)); } for (int i=0; i < 2; i++) { kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE)); } for (int i=10; i < 20; i++) { kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE)); } verifyResult(result, kvListExp, toLog, "Testing multiple CFs"); // Filters: ColumnRangeFilter and ColumnPrefixFilter get = new Get(ROW); get.setMaxResultsPerColumnFamily(3); get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true)); result = ht.get(get); kvListExp = new ArrayList<KeyValue>(); for (int i=2; i < 5; i++) { kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE)); } for (int i=2; i < 5; i++) { kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE)); } for (int i=2; i < 5; i++) { kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE)); } verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF"); get = new Get(ROW); get.setMaxResultsPerColumnFamily(7); get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1])); result = ht.get(get); kvListExp = new ArrayList<KeyValue>(); kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE)); for (int i=10; i < 16; i++) { kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE)); } verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF"); } /** * Test from client side for scan with maxResultPerCF set * * @throws Exception */ @Test public void testScanMaxResults() throws Exception { byte [] TABLE = Bytes.toBytes("testScanLimit"); byte [][] ROWS = HTestConst.makeNAscii(ROW, 2); byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3); byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10); HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES); Put put; Scan scan; Result result; boolean toLog = true; List<KeyValue> kvListExp, kvListScan; kvListExp = new ArrayList<KeyValue>(); for (int r=0; r < ROWS.length; r++) { put = new Put(ROWS[r]); for (int c=0; c < FAMILIES.length; c++) { for (int q=0; q < QUALIFIERS.length; q++) { KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE); put.add(kv); if (q < 4) { kvListExp.add(kv); } } } ht.put(put); } scan = new Scan(); scan.setMaxResultsPerColumnFamily(4); ResultScanner scanner = ht.getScanner(scan); kvListScan = new ArrayList<KeyValue>(); while ((result = scanner.next()) != null) { for (KeyValue kv : result.list()) { kvListScan.add(kv); } } result = new Result(kvListScan); verifyResult(result, kvListExp, toLog, "Testing scan with maxResults"); } /** * Test from client side for get with rowOffset * * @throws Exception */ @Test public void testGetRowOffset() throws Exception { byte [] TABLE = Bytes.toBytes("testGetRowOffset"); byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3); byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20); HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES); Get get; Put put; Result result; boolean toLog = true; List<KeyValue> kvListExp; // Insert one CF for row kvListExp = new ArrayList<KeyValue>(); put = new Put(ROW); for (int i=0; i < 10; i++) { KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE); put.add(kv); // skipping first two kvs if (i < 2) continue; kvListExp.add(kv); } ht.put(put); //setting offset to 2 get = new Get(ROW); get.setRowOffsetPerColumnFamily(2); result = ht.get(get); verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset"); //setting offset to 20 get = new Get(ROW); get.setRowOffsetPerColumnFamily(20); result = ht.get(get); kvListExp = new ArrayList<KeyValue>(); verifyResult(result, kvListExp, toLog, "Testing offset > #kvs"); //offset + maxResultPerCF get = new Get(ROW); get.setRowOffsetPerColumnFamily(4); get.setMaxResultsPerColumnFamily(5); result = ht.get(get); kvListExp = new ArrayList<KeyValue>(); for (int i=4; i < 9; i++) { kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE)); } verifyResult(result, kvListExp, toLog, "Testing offset + setMaxResultsPerCF"); // Filters: ColumnRangeFilter get = new Get(ROW); get.setRowOffsetPerColumnFamily(1); get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5], true)); result = ht.get(get); kvListExp = new ArrayList<KeyValue>(); kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE)); verifyResult(result, kvListExp, toLog, "Testing offset with CRF"); // Insert into two more CFs for row // 10 columns for CF2, 10 columns for CF1 for(int j=2; j > 0; j--) { put = new Put(ROW); for (int i=0; i < 10; i++) { KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE); put.add(kv); } ht.put(put); } get = new Get(ROW); get.setRowOffsetPerColumnFamily(4); get.setMaxResultsPerColumnFamily(2); get.addFamily(FAMILIES[1]); get.addFamily(FAMILIES[2]); result = ht.get(get); kvListExp = new ArrayList<KeyValue>(); //Exp: CF1:q4, q5, CF2: q4, q5 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE)); kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE)); verifyResult(result, kvListExp, toLog, "Testing offset + multiple CFs + maxResults"); } static void verifyResult(Result result, List<KeyValue> expKvList, boolean toLog, String msg) { LOG.info(msg); LOG.info("Expected count: " + expKvList.size()); LOG.info("Actual count: " + result.size()); if (expKvList.size() == 0) return; int i = 0; for (KeyValue kv : result.raw()) { if (i >= expKvList.size()) { break; // we will check the size later } KeyValue kvExp = expKvList.get(i++); if (toLog) { LOG.info("get kv is: " + kv.toString()); LOG.info("exp kv is: " + kvExp.toString()); } assertTrue("Not equal", kvExp.equals(kv)); } assertEquals(expKvList.size(), result.size()); } }