/*
* Copyright © 2014-2015 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package co.cask.cdap.data2.increment.hbase10;
import co.cask.cdap.common.conf.CConfiguration;
import co.cask.cdap.data.hbase.HBase10Test;
import co.cask.cdap.data2.dataset2.lib.table.hbase.HBaseTable;
import co.cask.cdap.data2.increment.hbase.IncrementHandlerState;
import co.cask.cdap.data2.util.TableId;
import co.cask.cdap.data2.util.hbase.HBaseTableUtil;
import co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory;
import co.cask.cdap.data2.util.hbase.HTableDescriptorBuilder;
import co.cask.cdap.proto.Id;
import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MockRegionServerServices;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import java.net.InetAddress;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* Tests for {@link IncrementSummingScanner} implementation.
*/
public class IncrementSummingScannerTest {
@ClassRule
public static final HBase10Test TEST_HBASE = new HBase10Test();
private static final byte[] TRUE = Bytes.toBytes(true);
private static Configuration conf;
private static CConfiguration cConf;
@BeforeClass
public static void setupBeforeClass() throws Exception {
conf = TEST_HBASE.getConfiguration();
cConf = CConfiguration.create();
}
@Test
public void testIncrementScanning() throws Exception {
TableId tableId = TableId.from(Id.Namespace.DEFAULT, "TestIncrementSummingScanner");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
// test handling of a single increment value alone
Put p = new Put(Bytes.toBytes("r1"));
p.add(familyBytes, columnBytes, Bytes.toBytes(3L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
Scan scan = new Scan();
RegionScanner scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
List<Cell> results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
Cell cell = results.get(0);
assertNotNull(cell);
assertEquals(3L, Bytes.toLong(cell.getValue()));
// test handling of a single total sum
p = new Put(Bytes.toBytes("r2"));
p.add(familyBytes, columnBytes, Bytes.toBytes(5L));
region.put(p);
scan = new Scan(Bytes.toBytes("r2"));
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(5L, Bytes.toLong(cell.getValue()));
// test handling of multiple increment values
long now = System.currentTimeMillis();
p = new Put(Bytes.toBytes("r3"));
for (int i = 0; i < 5; i++) {
p.add(familyBytes, columnBytes, now - i, Bytes.toBytes((long) (i + 1)));
}
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
scan = new Scan(Bytes.toBytes("r3"));
scan.setMaxVersions();
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(15L, Bytes.toLong(cell.getValue()));
// test handling of multiple increment values followed by a total sum, then other increments
now = System.currentTimeMillis();
p = new Put(Bytes.toBytes("r4"));
for (int i = 0; i < 3; i++) {
p.add(familyBytes, columnBytes, now - i, Bytes.toBytes(1L));
}
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
// this put will appear as a "total" sum prior to all the delta puts
p = new Put(Bytes.toBytes("r4"));
p.add(familyBytes, columnBytes, now - 5, Bytes.toBytes(5L));
region.put(p);
scan = new Scan(Bytes.toBytes("r4"));
scan.setMaxVersions();
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(8L, Bytes.toLong(cell.getValue()));
// test handling of an increment column followed by a non-increment column
p = new Put(Bytes.toBytes("r4"));
p.add(familyBytes, Bytes.toBytes("c2"), Bytes.toBytes("value"));
region.put(p);
scan = new Scan(Bytes.toBytes("r4"));
scan.setMaxVersions();
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(2, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(8L, Bytes.toLong(cell.getValue()));
cell = results.get(1);
assertNotNull(cell);
assertEquals("value", Bytes.toString(cell.getValue()));
// test handling of an increment column followed by a delete
now = System.currentTimeMillis();
Delete d = new Delete(Bytes.toBytes("r5"));
d.deleteColumn(familyBytes, columnBytes, now - 3);
region.delete(d);
p = new Put(Bytes.toBytes("r5"));
for (int i = 2; i >= 0; i--) {
p.add(familyBytes, columnBytes, now - i, Bytes.toBytes(1L));
}
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
scan = new Scan(Bytes.toBytes("r5"));
scan.setMaxVersions();
scan.setRaw(true);
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.COMPACT_RETAIN_DELETES);
results = Lists.newArrayList();
scanner.next(results);
// delete marker will not be returned for user scan
assertEquals(2, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(3L, Bytes.toLong(cell.getValue(), IncrementHandlerState.DELTA_MAGIC_PREFIX.length, 8));
// next cell should be the delete
cell = results.get(1);
assertTrue(CellUtil.isDelete(cell));
} finally {
region.close();
}
}
@Test
public void testFlushAndCompact() throws Exception {
TableId tableId = TableId.from(Id.Namespace.DEFAULT, "TestFlushAndCompact");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
// load an initial set of increments
long ts = System.currentTimeMillis();
byte[] row1 = Bytes.toBytes("row1");
for (int i = 0; i < 50; i++) {
Put p = new Put(row1);
p.add(familyBytes, columnBytes, ts, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
ts++;
region.put(p);
}
byte[] row2 = Bytes.toBytes("row2");
ts = System.currentTimeMillis();
// start with a full put
Put row2P = new Put(row2);
row2P.add(familyBytes, columnBytes, ts++, Bytes.toBytes(10L));
region.put(row2P);
for (int i = 0; i < 10; i++) {
Put p = new Put(row2);
p.add(familyBytes, columnBytes, ts++, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
}
// force a region flush
region.flushcache();
region.waitForFlushesAndCompactions();
Result r1 = region.get(new Get(row1));
assertNotNull(r1);
assertFalse(r1.isEmpty());
// row1 should have a full put aggregating all 50 incrments
Cell r1Cell = r1.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r1Cell);
assertEquals(50L, Bytes.toLong(r1Cell.getValue()));
Result r2 = region.get(new Get(row2));
assertNotNull(r2);
assertFalse(r2.isEmpty());
// row2 should have a full put aggregating prior put + 10 increments
Cell r2Cell = r2.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r2Cell);
assertEquals(20L, Bytes.toLong(r2Cell.getValue()));
// add 30 more increments to row2
for (int i = 0; i < 30; i++) {
Put p = new Put(row2);
p.add(familyBytes, columnBytes, ts++, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
}
// row2 should now have a full put aggregating prior 20 value + 30 increments
r2 = region.get(new Get(row2));
assertNotNull(r2);
assertFalse(r2.isEmpty());
r2Cell = r2.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r2Cell);
assertEquals(50L, Bytes.toLong(r2Cell.getValue()));
// force another region flush
region.flushcache();
region.waitForFlushesAndCompactions();
// add 100 more increments to row2
for (int i = 0; i < 100; i++) {
Put p = new Put(row2);
p.add(familyBytes, columnBytes, ts++, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
}
// row2 should now have a full put aggregating prior 50 value + 100 increments
r2 = region.get(new Get(row2));
assertNotNull(r2);
assertFalse(r2.isEmpty());
r2Cell = r2.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r2Cell);
assertEquals(150L, Bytes.toLong(r2Cell.getValue()));
} finally {
region.close();
}
}
@Test
public void testMultiColumnFlushAndCompact() throws Exception {
TableId tableId = TableId.from(Id.Namespace.DEFAULT, "testMultiColumnFlushAndCompact");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
byte[] columnBytes2 = Bytes.toBytes("c2");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
long now = 1;
byte[] row1 = Bytes.toBytes("row1");
byte[] row2 = Bytes.toBytes("row2");
// Initial put to row1,c2
Put row1P = new Put(row1);
row1P.add(familyBytes, columnBytes2, now - 1, Bytes.toBytes(5L));
region.put(row1P);
// Initial put to row2,c
Put row2P = new Put(row2);
row2P.add(familyBytes, columnBytes, now - 1, Bytes.toBytes(10L));
region.put(row2P);
// Generate some increments
long ts = now;
for (int i = 0; i < 50; i++) {
region.put(generateIncrementPut(familyBytes, columnBytes, row1, ts));
region.put(generateIncrementPut(familyBytes, columnBytes, row2, ts));
region.put(generateIncrementPut(familyBytes, columnBytes2, row1, ts));
ts++;
}
// First scanner represents flush scanner
RegionScanner scanner =
new IncrementSummingScanner(region, -1, region.getScanner(new Scan().setMaxVersions()),
ScanType.COMPACT_RETAIN_DELETES, now + 15, -1);
// Second scanner is a user scan, this is to help in easy asserts
scanner = new IncrementSummingScanner(region, -1, scanner, ScanType.USER_SCAN);
List<Cell> results = Lists.newArrayList();
assertTrue(scanner.next(results, 10));
assertEquals(2, results.size());
Cell cell = results.get(0);
assertNotNull(cell);
assertEquals("row1", Bytes.toString(cell.getRow()));
assertEquals("c", Bytes.toString(cell.getQualifier()));
assertEquals(50, Bytes.toLong(cell.getValue()));
cell = results.get(1);
assertNotNull(cell);
assertEquals("row1", Bytes.toString(cell.getRow()));
assertEquals("c2", Bytes.toString(cell.getQualifier()));
assertEquals(55, Bytes.toLong(cell.getValue()));
results.clear();
assertFalse(scanner.next(results, 10));
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals("row2", Bytes.toString(cell.getRow()));
assertEquals(60, Bytes.toLong(cell.getValue()));
} finally {
region.close();
}
}
@Test
public void testIncrementScanningWithBatchAndUVB() throws Exception {
TableId tableId = TableId.from(Id.Namespace.DEFAULT, "TestIncrementSummingScannerWithUpperVisibilityBound");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
long start = 0;
long now = start;
long counter1 = 0;
// adding 5 delta increments
for (int i = 0; i < 5; i++) {
Put p = new Put(Bytes.toBytes("r1"), now++);
p.add(familyBytes, columnBytes, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
counter1++;
}
// different combinations of uvb and limit (see batch test above)
// At least these cases we want to cover for batch:
// * batch=<not set> // unlimited by default
// * batch=1
// * batch size less than delta inc group size
// * batch size greater than delta inc group size
// * batch size is bigger than all delta incs available
// At least these cases we want to cover for uvb:
// * uvb=<not set> // 0
// * uvb less than max tx of delta inc
// * uvb greater than max tx of delta inc
// * multiple uvbs applied to simulate multiple flush & compactions
// Also: we want different combinations of batch limit & uvbs
for (int i = 0; i < 7; i++) {
for (int k = 0; k < 4; k++) {
long[] uvbs = new long[k];
for (int l = 0; l < uvbs.length; l++) {
uvbs[l] = start + (k + 1) * (l + 1);
}
verifyCounts(region, new Scan().setMaxVersions(), new long[]{counter1}, i > 0 ? i : -1, uvbs);
}
}
// Now test same with two groups of increments
int counter2 = 0;
for (int i = 0; i < 5; i++) {
Put p = new Put(Bytes.toBytes("r2"), now + i);
p.add(familyBytes, columnBytes, Bytes.toBytes(2L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
counter2 += 2;
}
for (int i = 0; i < 12; i++) {
for (int k = 0; k < 4; k++) {
long[] uvbs = new long[k];
for (int l = 0; l < uvbs.length; l++) {
uvbs[l] = start + (k + 1) * (l + 1);
}
verifyCounts(region, new Scan().setMaxVersions(), new long[]{counter1, counter2}, i > 0 ? i : -1, uvbs);
}
}
} finally {
region.close();
}
}
private Put generateIncrementPut(byte[] familyBytes, byte[] columnBytes, byte [] row, long ts) {
Put p = new Put(row);
p.add(familyBytes, columnBytes, ts, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
return p;
}
private void verifyCounts(HRegion region, Scan scan, long[] counts) throws Exception {
verifyCounts(region, scan, counts, -1);
}
private void verifyCounts(HRegion region, Scan scan, long[] counts, int batch) throws Exception {
RegionScanner scanner = new IncrementSummingScanner(region, batch, region.getScanner(scan), ScanType.USER_SCAN);
// init with false if loop will execute zero times
boolean hasMore = counts.length > 0;
for (long count : counts) {
List<Cell> results = Lists.newArrayList();
hasMore = scanner.next(results);
assertEquals(1, results.size());
Cell cell = results.get(0);
assertNotNull(cell);
assertEquals(count, Bytes.toLong(cell.getValue()));
}
assertFalse(hasMore);
}
private void verifyCounts(HRegion region, Scan scan, long[] counts, int batch, long[] upperVisBound)
throws Exception {
// The idea is to chain IncrementSummingScanner: first couple respect the upperVisBound and may produce multiple
// cells for single value. This is what happens during flush or compaction. Second one will mimic user scan over
// flushed or compacted: it should merge all delta increments appropriately.
RegionScanner scanner = region.getScanner(scan);
for (int i = 0; i < upperVisBound.length; i++) {
scanner = new IncrementSummingScanner(region, batch, scanner,
ScanType.COMPACT_RETAIN_DELETES, upperVisBound[i], -1);
}
scanner = new IncrementSummingScanner(region, batch, scanner, ScanType.USER_SCAN);
// init with false if loop will execute zero times
boolean hasMore = counts.length > 0;
for (long count : counts) {
List<Cell> results = Lists.newArrayList();
hasMore = scanner.next(results);
assertEquals(1, results.size());
Cell cell = results.get(0);
assertNotNull(cell);
assertEquals(count, Bytes.toLong(cell.getValue()));
}
assertFalse(hasMore);
}
private HRegion createRegion(TableId tableId, byte[] family) throws Exception {
return createRegion(conf, cConf, tableId, new HColumnDescriptor(family));
}
static HRegion createRegion(Configuration hConf, CConfiguration cConf, TableId tableId,
HColumnDescriptor cfd) throws Exception {
HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
HTableDescriptorBuilder htd = tableUtil.buildHTableDescriptor(tableId);
cfd.setMaxVersions(Integer.MAX_VALUE);
cfd.setKeepDeletedCells(true);
htd.addFamily(cfd);
htd.addCoprocessor(IncrementHandler.class.getName());
HTableDescriptor desc = htd.build();
String tableName = desc.getNameAsString();
Path tablePath = new Path("/tmp/" + tableName);
Path hlogPath = new Path("/tmp/hlog-" + tableName);
FileSystem fs = FileSystem.get(hConf);
assertTrue(fs.mkdirs(tablePath));
WALFactory walFactory = new WALFactory(hConf, null, hlogPath.toString());
WAL hLog = walFactory.getWAL(new byte[]{1});
HRegionInfo regionInfo = new HRegionInfo(desc.getTableName());
HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(hConf, fs, tablePath, regionInfo);
return new HRegion(regionFS, hLog, hConf, desc,
new LocalRegionServerServices(hConf, ServerName.valueOf(
InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
private static class LocalRegionServerServices extends MockRegionServerServices {
private final ServerName serverName;
public LocalRegionServerServices(Configuration conf, ServerName serverName) {
super(conf);
this.serverName = serverName;
}
@Override
public ServerName getServerName() {
return serverName;
}
}
}