/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.hadoop.hive.metastore.hbase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Decimal; import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.FunctionType; import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; import org.apache.hadoop.hive.metastore.api.HiveObjectRef; import org.apache.hadoop.hive.metastore.api.HiveObjectType; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; import org.apache.hadoop.hive.metastore.api.ResourceType; import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; import org.apache.hadoop.hive.metastore.api.Table; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.TreeSet; /** * Integration tests with HBase Mini-cluster for HBaseStore */ public class TestHBaseStoreIntegration extends HBaseIntegrationTests { private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreIntegration.class.getName()); @Rule public ExpectedException thrown = ExpectedException.none(); @BeforeClass public static void startup() throws Exception { HBaseIntegrationTests.startMiniCluster(); } @AfterClass public static void shutdown() throws Exception { HBaseIntegrationTests.shutdownMiniCluster(); } @Before public void setup() throws IOException { setupConnection(); setupHBaseStore(); } @Test public void createDb() throws Exception { String dbname = "mydb"; Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters); store.createDatabase(db); Database d = store.getDatabase("mydb"); Assert.assertEquals(dbname, d.getName()); Assert.assertEquals("no description", d.getDescription()); Assert.assertEquals("file:///tmp", d.getLocationUri()); } @Test public void dropDb() throws Exception { String dbname = "anotherdb"; Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters); store.createDatabase(db); Database d = store.getDatabase(dbname); Assert.assertNotNull(d); store.dropDatabase(dbname); thrown.expect(NoSuchObjectException.class); store.getDatabase(dbname); } @Test public void getAllDbs() throws Exception { String[] dbNames = new String[3]; for (int i = 0; i < dbNames.length; i++) { dbNames[i] = "db" + i; Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters); store.createDatabase(db); } List<String> dbs = store.getAllDatabases(); Assert.assertEquals(3, dbs.size()); String[] namesFromStore = dbs.toArray(new String[3]); Arrays.sort(namesFromStore); Assert.assertArrayEquals(dbNames, namesFromStore); } @Test public void getDbsRegex() throws Exception { String[] dbNames = new String[3]; for (int i = 0; i < dbNames.length; i++) { dbNames[i] = "db" + i; Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters); store.createDatabase(db); } List<String> dbs = store.getDatabases("db1|db2"); Assert.assertEquals(2, dbs.size()); String[] namesFromStore = dbs.toArray(new String[2]); Arrays.sort(namesFromStore); Assert.assertArrayEquals(Arrays.copyOfRange(dbNames, 1, 3), namesFromStore); dbs = store.getDatabases("db*"); Assert.assertEquals(3, dbs.size()); namesFromStore = dbs.toArray(new String[3]); Arrays.sort(namesFromStore); Assert.assertArrayEquals(dbNames, namesFromStore); } @Test public void getFuncsRegex() throws Exception { String dbname = "default"; int now = (int)(System.currentTimeMillis()/1000); String[] funcNames = new String[3]; for (int i = 0; i < funcNames.length; i++) { funcNames[i] = "func" + i; store.createFunction(new Function(funcNames[i], dbname, "o.a.h.h.myfunc", "me", PrincipalType.USER, now, FunctionType.JAVA, Arrays.asList(new ResourceUri(ResourceType.JAR, "file:/tmp/somewhere")))); } List<String> funcs = store.getFunctions(dbname, "func1|func2"); Assert.assertEquals(2, funcs.size()); String[] namesFromStore = funcs.toArray(new String[2]); Arrays.sort(namesFromStore); Assert.assertArrayEquals(Arrays.copyOfRange(funcNames, 1, 3), namesFromStore); funcs = store.getFunctions(dbname, "func*"); Assert.assertEquals(3, funcs.size()); namesFromStore = funcs.toArray(new String[3]); Arrays.sort(namesFromStore); Assert.assertArrayEquals(funcNames, namesFromStore); funcs = store.getFunctions("nosuchdb", "func*"); Assert.assertEquals(0, funcs.size()); } @Test public void createTable() throws Exception { int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); Table table = new Table("mytable", "default", "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null); store.createTable(table); Table t = store.getTable("default", "mytable"); Assert.assertEquals(1, t.getSd().getColsSize()); Assert.assertEquals("col1", t.getSd().getCols().get(0).getName()); Assert.assertEquals("int", t.getSd().getCols().get(0).getType()); Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment()); Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName()); Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib()); Assert.assertEquals("file:/tmp", t.getSd().getLocation()); Assert.assertEquals("input", t.getSd().getInputFormat()); Assert.assertEquals("output", t.getSd().getOutputFormat()); Assert.assertEquals("me", t.getOwner()); Assert.assertEquals("default", t.getDbName()); Assert.assertEquals("mytable", t.getTableName()); } @Test public void alterTable() throws Exception { String tableName = "alttable"; int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null); store.createTable(table); startTime += 10; table.setLastAccessTime(startTime); LOG.debug("XXX alter table test"); store.alterTable("default", tableName, table); Table t = store.getTable("default", tableName); LOG.debug("Alter table time " + t.getLastAccessTime()); Assert.assertEquals(1, t.getSd().getColsSize()); Assert.assertEquals("col1", t.getSd().getCols().get(0).getName()); Assert.assertEquals("int", t.getSd().getCols().get(0).getType()); Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment()); Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName()); Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib()); Assert.assertEquals("file:/tmp", t.getSd().getLocation()); Assert.assertEquals("input", t.getSd().getInputFormat()); Assert.assertEquals("output", t.getSd().getOutputFormat()); Assert.assertEquals("me", t.getOwner()); Assert.assertEquals("default", t.getDbName()); Assert.assertEquals(tableName, t.getTableName()); Assert.assertEquals(startTime, t.getLastAccessTime()); } @Test public void getAllTables() throws Exception { String dbNames[] = new String[]{"db0", "db1"}; // named to match getAllDbs so we get the // right number of databases in that test. String tableNames[] = new String[]{"curly", "larry", "moe"}; for (int i = 0; i < dbNames.length; i++) { store.createDatabase(new Database(dbNames[i], "no description", "file:///tmp", emptyParameters)); } for (int i = 0; i < dbNames.length; i++) { for (int j = 0; j < tableNames.length; j++) { int startTime = (int) (System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); Table table = new Table(tableNames[j], dbNames[i], "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null); store.createTable(table); } } List<String> fetchedNames = store.getAllTables(dbNames[0]); Assert.assertEquals(3, fetchedNames.size()); String[] sortedFetchedNames = fetchedNames.toArray(new String[fetchedNames.size()]); Arrays.sort(sortedFetchedNames); Assert.assertArrayEquals(tableNames, sortedFetchedNames); List<String> regexNames = store.getTables(dbNames[0], "*y"); Assert.assertEquals(2, regexNames.size()); String[] sortedRegexNames = regexNames.toArray(new String[regexNames.size()]); Arrays.sort(sortedRegexNames); Assert.assertArrayEquals(Arrays.copyOfRange(tableNames, 0, 2), sortedRegexNames); List<Table> fetchedTables = store.getTableObjectsByName(dbNames[1], Arrays.asList(Arrays.copyOfRange(tableNames, 1, 3))); Assert.assertEquals(2, fetchedTables.size()); sortedFetchedNames = new String[fetchedTables.size()]; for (int i = 0; i < fetchedTables.size(); i++) { sortedFetchedNames[i] = fetchedTables.get(i).getTableName(); } Arrays.sort(sortedFetchedNames); Assert.assertArrayEquals(Arrays.copyOfRange(tableNames, 1, 3), sortedFetchedNames); } @Test public void dropTable() throws Exception { String tableName = "dtable"; int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null); store.createTable(table); Table t = store.getTable("default", tableName); Assert.assertNotNull(t); store.dropTable("default", tableName); Assert.assertNull(store.getTable("default", tableName)); } @Test public void createPartition() throws Exception { String dbName = "default"; String tableName = "myparttable"; int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("pc", "string", "")); Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null); store.createTable(table); List<String> vals = new ArrayList<String>(); vals.add("fred"); StorageDescriptor psd = new StorageDescriptor(sd); psd.setLocation("file:/tmp/pc=fred"); Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters); store.addPartition(part); Partition p = store.getPartition(dbName, tableName, vals); Assert.assertEquals(1, p.getSd().getColsSize()); Assert.assertEquals("col1", p.getSd().getCols().get(0).getName()); Assert.assertEquals("int", p.getSd().getCols().get(0).getType()); Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment()); Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName()); Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib()); Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation()); Assert.assertEquals("input", p.getSd().getInputFormat()); Assert.assertEquals("output", p.getSd().getOutputFormat()); Assert.assertEquals(dbName, p.getDbName()); Assert.assertEquals(tableName, p.getTableName()); Assert.assertEquals(1, p.getValuesSize()); Assert.assertEquals("fred", p.getValues().get(0)); } @Test public void addPartitions() throws Exception { String dbName = "default"; String tableName = "addParts"; int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("pc", "string", "")); Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null); store.createTable(table); List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan"); List<Partition> partitions = new ArrayList<Partition>(); for (String val : partVals) { List<String> vals = new ArrayList<String>(); vals.add(val); StorageDescriptor psd = new StorageDescriptor(sd); psd.setLocation("file:/tmp/pc=" + val); Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters); partitions.add(part); } store.addPartitions(dbName, tableName, partitions); List<String> partNames = store.listPartitionNames(dbName, tableName, (short) -1); Assert.assertEquals(5, partNames.size()); String[] names = partNames.toArray(new String[partNames.size()]); Arrays.sort(names); String[] canonicalNames = partVals.toArray(new String[partVals.size()]); for (int i = 0; i < canonicalNames.length; i++) canonicalNames[i] = "pc=" + canonicalNames[i]; Assert.assertArrayEquals(canonicalNames, names); } @Test public void alterPartitions() throws Exception { String dbName = "default"; String tableName = "alterParts"; int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("pc", "string", "")); Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null); store.createTable(table); List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan"); List<Partition> partitions = new ArrayList<Partition>(); List<List<String>> allVals = new ArrayList<List<String>>(); for (String val : partVals) { List<String> vals = new ArrayList<String>(); allVals.add(vals); vals.add(val); StorageDescriptor psd = new StorageDescriptor(sd); psd.setLocation("file:/tmp/pc=" + val); Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters); partitions.add(part); } store.addPartitions(dbName, tableName, partitions); for (Partition p : partitions) p.setLastAccessTime(startTime + 10); store.alterPartitions(dbName, tableName, allVals, partitions); partitions = store.getPartitions(dbName, tableName, -1); for (Partition part : partitions) { Assert.assertEquals(startTime + 10, part.getLastAccessTime()); } } @Test public void getPartitions() throws Exception { String dbName = "default"; String tableName = "manyParts"; int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("pc", "string", "")); Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null); store.createTable(table); List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan"); for (String val : partVals) { List<String> vals = new ArrayList<String>(); vals.add(val); StorageDescriptor psd = new StorageDescriptor(sd); psd.setLocation("file:/tmp/pc=" + val); Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters); store.addPartition(part); Partition p = store.getPartition(dbName, tableName, vals); Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation()); } List<Partition> parts = store.getPartitions(dbName, tableName, -1); Assert.assertEquals(5, parts.size()); String[] pv = new String[5]; for (int i = 0; i < 5; i++) pv[i] = parts.get(i).getValues().get(0); Arrays.sort(pv); Assert.assertArrayEquals(pv, partVals.toArray(new String[5])); } @Test public void listPartitions() throws Exception { String dbName = "default"; String tableName = "listParts"; int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("pc", "string", "")); partCols.add(new FieldSchema("region", "string", "")); Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null); store.createTable(table); String[][] partVals = new String[][]{{"today", "north america"}, {"tomorrow", "europe"}}; for (String[] pv : partVals) { List<String> vals = new ArrayList<String>(); for (String v : pv) vals.add(v); StorageDescriptor psd = new StorageDescriptor(sd); psd.setLocation("file:/tmp/pc=" + pv[0] + "/region=" + pv[1]); Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters); store.addPartition(part); } List<String> names = store.listPartitionNames(dbName, tableName, (short) -1); Assert.assertEquals(2, names.size()); String[] resultNames = names.toArray(new String[names.size()]); Arrays.sort(resultNames); Assert.assertArrayEquals(resultNames, new String[]{"pc=today/region=north america", "pc=tomorrow/region=europe"}); List<Partition> parts = store.getPartitionsByNames(dbName, tableName, names); Assert.assertArrayEquals(partVals[0], parts.get(0).getValues().toArray(new String[2])); Assert.assertArrayEquals(partVals[1], parts.get(1).getValues().toArray(new String[2])); store.dropPartitions(dbName, tableName, names); List<Partition> afterDropParts = store.getPartitions(dbName, tableName, -1); Assert.assertEquals(0, afterDropParts.size()); } @Test public void listPartitionsWithPs() throws Exception { String dbName = "default"; String tableName = "listPartitionsWithPs"; int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("ds", "string", "")); partCols.add(new FieldSchema("region", "string", "")); Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null); store.createTable(table); String[][] partVals = new String[][]{{"today", "north america"}, {"today", "europe"}, {"tomorrow", "north america"}, {"tomorrow", "europe"}}; for (String[] pv : partVals) { List<String> vals = new ArrayList<String>(); for (String v : pv) vals.add(v); StorageDescriptor psd = new StorageDescriptor(sd); psd.setLocation("file:/tmp/ds=" + pv[0] + "/region=" + pv[1]); Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters); store.addPartition(part); } // We only test listPartitionNamesPs since it calls listPartitionsPsWithAuth anyway. // Test the case where we completely specify the partition List<String> partitionNames = store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0]), (short) -1); Assert.assertEquals(1, partitionNames.size()); Assert.assertEquals("ds=today/region=north america", partitionNames.get(0)); // Leave off the last value of the partition partitionNames = store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0][0]), (short)-1); Assert.assertEquals(2, partitionNames.size()); String[] names = partitionNames.toArray(new String[partitionNames.size()]); Arrays.sort(names); Assert.assertArrayEquals(new String[] {"ds=today/region=europe", "ds=today/region=north america"}, names); // Put a star in the last value of the partition partitionNames = store.listPartitionNamesPs(dbName, tableName, Arrays.asList("today", "*"), (short)-1); Assert.assertEquals(2, partitionNames.size()); names = partitionNames.toArray(new String[partitionNames.size()]); Arrays.sort(names); Assert.assertArrayEquals(new String[] {"ds=today/region=europe", "ds=today/region=north america"}, names); // Put a star in the first value of the partition partitionNames = store.listPartitionNamesPs(dbName, tableName, Arrays.asList("*", "europe"), (short)-1); Assert.assertEquals(2, partitionNames.size()); names = partitionNames.toArray(new String[partitionNames.size()]); Arrays.sort(names); Assert.assertArrayEquals(new String[] {"ds=today/region=europe", "ds=tomorrow/region=europe"}, names); } @Test public void getPartitionsByFilter() throws Exception { String dbName = "default"; String tableName = "getPartitionsByFilter"; int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("ds", "string", "")); partCols.add(new FieldSchema("region", "string", "")); Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null); store.createTable(table); String[][] partVals = new String[][]{{"20010101", "north america"}, {"20010101", "europe"}, {"20010102", "north america"}, {"20010102", "europe"}, {"20010103", "north america"}}; for (String[] pv : partVals) { List<String> vals = new ArrayList<String>(); for (String v : pv) vals.add(v); StorageDescriptor psd = new StorageDescriptor(sd); psd.setLocation("file:/tmp/ds=" + pv[0] + "/region=" + pv[1]); Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters); store.addPartition(part); } // We only test getPartitionsByFilter since it calls same code as getPartitionsByExpr anyway. // Test the case where we completely specify the partition List<Partition> parts = null; parts = store.getPartitionsByFilter(dbName, tableName, "ds > '20010101'", (short) -1); checkPartVals(parts, "[20010102, north america]", "[20010102, europe]", "[20010103, north america]"); parts = store.getPartitionsByFilter(dbName, tableName, "ds >= '20010102'", (short) -1); checkPartVals(parts, "[20010102, north america]", "[20010102, europe]", "[20010103, north america]"); parts = store.getPartitionsByFilter(dbName, tableName, "ds >= '20010102' and region = 'europe' ", (short) -1); // filtering on first partition is only implemented as of now, so it will // not filter on region checkPartVals(parts, "[20010102, north america]", "[20010102, europe]", "[20010103, north america]"); parts = store.getPartitionsByFilter(dbName, tableName, "ds >= '20010101' and ds < '20010102'", (short) -1); checkPartVals(parts,"[20010101, north america]", "[20010101, europe]"); parts = store.getPartitionsByFilter(dbName, tableName, "ds = '20010102' or ds < '20010103'", (short) -1); checkPartVals(parts, "[20010101, north america]", "[20010101, europe]", "[20010102, north america]", "[20010102, europe]"); // test conversion to DNF parts = store.getPartitionsByFilter(dbName, tableName, "ds = '20010102' and (ds = '20010102' or region = 'europe')", (short) -1); // filtering on first partition is only implemented as of now, so it will not filter on region checkPartVals(parts, "[20010102, north america]", "[20010102, europe]"); parts = store.getPartitionsByFilter(dbName, tableName, "region = 'europe'", (short) -1); // filtering on first partition is only implemented as of now, so it will not filter on region checkPartVals(parts, "[20010101, north america]", "[20010101, europe]", "[20010102, north america]", "[20010102, europe]", "[20010103, north america]"); } /** * Check if the given partitions have same values as given partitions value strings * @param parts given partitions * @param expectedPartVals */ private void checkPartVals(List<Partition> parts, String ... expectedPartVals) { Assert.assertEquals("number of partitions", expectedPartVals.length, parts.size()); Set<String> partValStrings = new TreeSet<String>(); for(Partition part : parts) { partValStrings.add(part.getValues().toString()); } partValStrings.equals(new TreeSet(Arrays.asList(expectedPartVals))); } @Test public void dropPartition() throws Exception { String dbName = "default"; String tableName = "myparttable2"; int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("pc", "string", "")); Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null); store.createTable(table); List<String> vals = Arrays.asList("fred"); StorageDescriptor psd = new StorageDescriptor(sd); psd.setLocation("file:/tmp/pc=fred"); Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters); store.addPartition(part); Assert.assertNotNull(store.getPartition(dbName, tableName, vals)); store.dropPartition(dbName, tableName, vals); thrown.expect(NoSuchObjectException.class); store.getPartition(dbName, tableName, vals); } @Test public void createRole() throws Exception { int now = (int)System.currentTimeMillis()/1000; String roleName = "myrole"; store.addRole(roleName, "me"); Role r = store.getRole(roleName); Assert.assertEquals(roleName, r.getRoleName()); Assert.assertEquals("me", r.getOwnerName()); Assert.assertTrue(now <= r.getCreateTime()); } @Test public void dropRole() throws Exception { String roleName = "anotherrole"; store.addRole(roleName, "me"); Role r = store.getRole(roleName); Assert.assertEquals(roleName, r.getRoleName()); store.removeRole(roleName); thrown.expect(NoSuchObjectException.class); store.getRole(roleName); } @Test public void grantRevokeRoles() throws Exception { int now = (int)(System.currentTimeMillis()/1000); String roleName1 = "role1"; store.addRole(roleName1, "me"); String roleName2 = "role2"; store.addRole(roleName2, "me"); Role role1 = store.getRole(roleName1); Role role2 = store.getRole(roleName2); store.grantRole(role1, "fred", PrincipalType.USER, "bob", PrincipalType.USER, false); store.grantRole(role2, roleName1, PrincipalType.ROLE, "admin", PrincipalType.ROLE, true); store.grantRole(role2, "fred", PrincipalType.USER, "admin", PrincipalType.ROLE, false); List<Role> roles = store.listRoles("fred", PrincipalType.USER); Assert.assertEquals(3, roles.size()); boolean sawRole1 = false, sawRole2 = false, sawPublic = false; for (Role role : roles) { if (role.getRoleName().equals(roleName1)) { sawRole1 = true; } else if (role.getRoleName().equals(roleName2)) { sawRole2 = true; } else if (role.getRoleName().equals(HiveMetaStore.PUBLIC)) { sawPublic = true; } else { Assert.fail("Unknown role name " + role.getRoleName()); } } Assert.assertTrue(sawRole1 && sawRole2 && sawPublic); roles = store.listRoles("fred", PrincipalType.ROLE); Assert.assertEquals(0, roles.size()); roles = store.listRoles(roleName1, PrincipalType.ROLE); Assert.assertEquals(1, roles.size()); Role role = roles.get(0); Assert.assertEquals(roleName2, role.getRoleName()); // Test listing all members in a role List<RolePrincipalGrant> grants = store.listRoleMembers(roleName1); Assert.assertEquals(1, grants.size()); Assert.assertEquals("fred", grants.get(0).getPrincipalName()); Assert.assertEquals(PrincipalType.USER, grants.get(0).getPrincipalType()); Assert.assertTrue("Expected grant time of " + now + " got " + grants.get(0).getGrantTime(), grants.get(0).getGrantTime() >= now); Assert.assertEquals("bob", grants.get(0).getGrantorName()); Assert.assertEquals(PrincipalType.USER, grants.get(0).getGrantorPrincipalType()); Assert.assertFalse(grants.get(0).isGrantOption()); grants = store.listRoleMembers(roleName2); Assert.assertEquals(2, grants.size()); boolean sawFred = false; sawRole1 = false; for (RolePrincipalGrant m : grants) { if ("fred".equals(m.getPrincipalName())) sawFred = true; else if (roleName1.equals(m.getPrincipalName())) sawRole1 = true; else Assert.fail("Unexpected principal " + m.getPrincipalName()); } Assert.assertTrue(sawFred && sawRole1); // Revoke a role with grant option, make sure it just goes to no grant option store.revokeRole(role2, roleName1, PrincipalType.ROLE, true); roles = store.listRoles(roleName1, PrincipalType.ROLE); Assert.assertEquals(1, roles.size()); Assert.assertEquals(roleName2, roles.get(0).getRoleName()); grants = store.listRoleMembers(roleName1); Assert.assertFalse(grants.get(0).isGrantOption()); // Drop a role, make sure it is properly removed from the map store.removeRole(roleName1); roles = store.listRoles("fred", PrincipalType.USER); Assert.assertEquals(2, roles.size()); sawRole2 = sawPublic = false; for (Role m : roles) { if (m.getRoleName().equals(roleName2)) sawRole2 = true; else if (m.getRoleName().equals(HiveMetaStore.PUBLIC)) sawPublic = true; else Assert.fail("Unknown role " + m.getRoleName()); } Assert.assertTrue(sawRole2 && sawPublic); roles = store.listRoles(roleName1, PrincipalType.ROLE); Assert.assertEquals(0, roles.size()); // Revoke a role without grant option, make sure it goes away store.revokeRole(role2, "fred", PrincipalType.USER, false); roles = store.listRoles("fred", PrincipalType.USER); Assert.assertEquals(1, roles.size()); Assert.assertEquals(HiveMetaStore.PUBLIC, roles.get(0).getRoleName()); } @Test public void userToRoleMap() throws Exception { String roleName1 = "utrm1"; store.addRole(roleName1, "me"); String roleName2 = "utrm2"; store.addRole(roleName2, "me"); String user1 = "wilma"; String user2 = "betty"; Role role1 = store.getRole(roleName1); Role role2 = store.getRole(roleName2); store.grantRole(role1, user1, PrincipalType.USER, "bob", PrincipalType.USER, false); store.grantRole(role1, roleName2, PrincipalType.ROLE, "admin", PrincipalType.ROLE, true); List<String> roles = HBaseReadWrite.getInstance().getUserRoles(user1); Assert.assertEquals(2, roles.size()); String[] roleNames = roles.toArray(new String[roles.size()]); Arrays.sort(roleNames); Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames); store.grantRole(role2, user1, PrincipalType.USER, "admin", PrincipalType.ROLE, false); store.grantRole(role1, user2, PrincipalType.USER, "bob", PrincipalType.USER, false); HBaseReadWrite.setConf(conf); roles = HBaseReadWrite.getInstance().getUserRoles(user2); Assert.assertEquals(2, roles.size()); roleNames = roles.toArray(new String[roles.size()]); Arrays.sort(roleNames); Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames); store.revokeRole(role1, roleName2, PrincipalType.ROLE, false); // user1 should still have both roles since she was granted into role1 specifically. user2 // should only have role2 now since role2 was revoked from role1. roles = HBaseReadWrite.getInstance().getUserRoles(user1); Assert.assertEquals(2, roles.size()); roleNames = roles.toArray(new String[roles.size()]); Arrays.sort(roleNames); Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames); roles = HBaseReadWrite.getInstance().getUserRoles(user2); Assert.assertEquals(1, roles.size()); Assert.assertEquals(roleName1, roles.get(0)); } @Test public void userToRoleMapOnDrop() throws Exception { String roleName1 = "utrmod1"; store.addRole(roleName1, "me"); String roleName2 = "utrmod2"; store.addRole(roleName2, "me"); String user1 = "pebbles"; String user2 = "bam-bam"; Role role1 = store.getRole(roleName1); Role role2 = store.getRole(roleName2); store.grantRole(role1, user1, PrincipalType.USER, "bob", PrincipalType.USER, false); store.grantRole(role1, roleName2, PrincipalType.ROLE, "admin", PrincipalType.ROLE, true); store.grantRole(role1, user2, PrincipalType.USER, "bob", PrincipalType.USER, false); List<String> roles = HBaseReadWrite.getInstance().getUserRoles(user2); Assert.assertEquals(2, roles.size()); String[] roleNames = roles.toArray(new String[roles.size()]); Arrays.sort(roleNames); Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames); store.removeRole(roleName2); HBaseReadWrite.setConf(conf); roles = HBaseReadWrite.getInstance().getUserRoles(user1); Assert.assertEquals(1, roles.size()); Assert.assertEquals(roleName1, roles.get(0)); roles = HBaseReadWrite.getInstance().getUserRoles(user2); Assert.assertEquals(1, roles.size()); Assert.assertEquals(roleName1, roles.get(0)); } @Test public void grantRevokeGlobalPrivileges() throws Exception { doGrantRevoke(HiveObjectType.GLOBAL, null, null, new String[] {"grpg1", "grpg2"}, new String[] {"bugs", "elmer", "daphy", "wiley"}); } @Test public void grantRevokeDbPrivileges() throws Exception { String dbName = "grdbp_db"; try { Database db = new Database(dbName, "no description", "file:///tmp", emptyParameters); store.createDatabase(db); doGrantRevoke(HiveObjectType.DATABASE, dbName, null, new String[] {"grdbp_role1", "grdbp_role2"}, new String[] {"fred", "barney", "wilma", "betty"}); } finally { store.dropDatabase(dbName); } } @Test public void grantRevokeTablePrivileges() throws Exception { String dbName = "grtp_db"; String tableName = "grtp_table"; try { Database db = new Database(dbName, "no description", "file:///tmp", emptyParameters); store.createDatabase(db); int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null); store.createTable(table); doGrantRevoke(HiveObjectType.TABLE, dbName, tableName, new String[] {"grtp_role1", "grtp_role2"}, new String[] {"batman", "robin", "superman", "wonderwoman"}); } finally { if (store.getTable(dbName, tableName) != null) store.dropTable(dbName, tableName); store.dropDatabase(dbName); } } private void doGrantRevoke(HiveObjectType objectType, String dbName, String tableName, String[] roleNames, String[] userNames) throws Exception { store.addRole(roleNames[0], "me"); store.addRole(roleNames[1], "me"); int now = (int)(System.currentTimeMillis() / 1000); Role role1 = store.getRole(roleNames[0]); Role role2 = store.getRole(roleNames[1]); store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false); store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true); store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false); List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>(); HiveObjectRef hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null); PrivilegeGrantInfo grantInfo = new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false); HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER, grantInfo); privileges.add(hop); hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null); grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true); hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo); privileges.add(hop); hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null); grantInfo = new PrivilegeGrantInfo("exec", now, "me", PrincipalType.USER, false); hop = new HiveObjectPrivilege(hiveObjRef, roleNames[1], PrincipalType.ROLE, grantInfo); privileges.add(hop); hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null); grantInfo = new PrivilegeGrantInfo("create", now, "me", PrincipalType.USER, true); hop = new HiveObjectPrivilege(hiveObjRef, userNames[2], PrincipalType.USER, grantInfo); privileges.add(hop); hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null); grantInfo = new PrivilegeGrantInfo("create2", now, "me", PrincipalType.USER, true); hop = new HiveObjectPrivilege(hiveObjRef, userNames[2], PrincipalType.USER, grantInfo); privileges.add(hop); PrivilegeBag pBag = new PrivilegeBag(privileges); store.grantPrivileges(pBag); PrincipalPrivilegeSet pps = getPPS(objectType, dbName, tableName, userNames[0]); Assert.assertEquals(1, pps.getUserPrivilegesSize()); Assert.assertEquals(1, pps.getUserPrivileges().get(userNames[0]).size()); grantInfo = pps.getUserPrivileges().get(userNames[0]).get(0); Assert.assertEquals("read", grantInfo.getPrivilege()); Assert.assertTrue(now <= grantInfo.getCreateTime()); Assert.assertEquals("me", grantInfo.getGrantor()); Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType()); Assert.assertFalse(grantInfo.isGrantOption()); Assert.assertEquals(2, pps.getRolePrivilegesSize()); Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[0]).size()); grantInfo = pps.getRolePrivileges().get(roleNames[0]).get(0); Assert.assertEquals("write", grantInfo.getPrivilege()); Assert.assertTrue(now <= grantInfo.getCreateTime()); Assert.assertEquals("me", grantInfo.getGrantor()); Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType()); Assert.assertTrue(grantInfo.isGrantOption()); Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[1]).size()); grantInfo = pps.getRolePrivileges().get(roleNames[1]).get(0); Assert.assertEquals("exec", grantInfo.getPrivilege()); Assert.assertTrue(now <= grantInfo.getCreateTime()); Assert.assertEquals("me", grantInfo.getGrantor()); Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType()); Assert.assertFalse(grantInfo.isGrantOption()); pps = getPPS(objectType, dbName, tableName, userNames[1]); Assert.assertEquals(0, pps.getUserPrivilegesSize()); Assert.assertEquals(1, pps.getRolePrivilegesSize()); Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[1]).size()); grantInfo = pps.getRolePrivileges().get(roleNames[1]).get(0); Assert.assertEquals("exec", grantInfo.getPrivilege()); Assert.assertTrue(now <= grantInfo.getCreateTime()); Assert.assertEquals("me", grantInfo.getGrantor()); Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType()); Assert.assertFalse(grantInfo.isGrantOption()); pps = getPPS(objectType, dbName, tableName, userNames[2]); Assert.assertEquals(1, pps.getUserPrivilegesSize()); Assert.assertEquals(2, pps.getUserPrivileges().get(userNames[2]).size()); Assert.assertEquals(0, pps.getRolePrivilegesSize()); pps = getPPS(objectType, dbName, tableName, userNames[3]); Assert.assertEquals(0, pps.getUserPrivilegesSize()); Assert.assertEquals(0, pps.getRolePrivilegesSize()); // Test that removing role removes the role grants store.removeRole(roleNames[1]); checkRoleRemovedFromAllPrivileges(objectType, dbName, tableName, roleNames[1]); pps = getPPS(objectType, dbName, tableName, userNames[0]); Assert.assertEquals(1, pps.getRolePrivilegesSize()); Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[0]).size()); pps = getPPS(objectType, dbName, tableName, userNames[1]); Assert.assertEquals(0, pps.getRolePrivilegesSize()); // Test that revoking with grant option = true just removes grant option privileges.clear(); hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null); grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true); hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo); privileges.add(hop); hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null); grantInfo = new PrivilegeGrantInfo("create2", now, "me", PrincipalType.USER, true); hop = new HiveObjectPrivilege(hiveObjRef, userNames[2], PrincipalType.USER, grantInfo); privileges.add(hop); pBag = new PrivilegeBag(privileges); store.revokePrivileges(pBag, true); pps = getPPS(objectType, dbName, tableName, userNames[0]); Assert.assertEquals(1, pps.getRolePrivilegesSize()); Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[0]).size()); grantInfo = pps.getRolePrivileges().get(roleNames[0]).get(0); Assert.assertEquals("write", grantInfo.getPrivilege()); Assert.assertTrue(now <= grantInfo.getCreateTime()); Assert.assertEquals("me", grantInfo.getGrantor()); Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType()); Assert.assertFalse(grantInfo.isGrantOption()); pps = getPPS(objectType, dbName, tableName, userNames[2]); Assert.assertEquals(1, pps.getUserPrivilegesSize()); Assert.assertEquals(2, pps.getUserPrivileges().get(userNames[2]).size()); for (PrivilegeGrantInfo pgi : pps.getUserPrivileges().get(userNames[2])) { if (pgi.getPrivilege().equals("create")) Assert.assertTrue(pgi.isGrantOption()); else if (pgi.getPrivilege().equals("create2")) Assert.assertFalse(pgi.isGrantOption()); else Assert.fail("huh?"); } // Test revoking revokes store.revokePrivileges(pBag, false); pps = getPPS(objectType, dbName, tableName, userNames[0]); Assert.assertEquals(1, pps.getUserPrivilegesSize()); Assert.assertEquals(1, pps.getRolePrivilegesSize()); Assert.assertEquals(0, pps.getRolePrivileges().get(roleNames[0]).size()); pps = getPPS(objectType, dbName, tableName, userNames[2]); Assert.assertEquals(1, pps.getUserPrivilegesSize()); Assert.assertEquals(1, pps.getUserPrivileges().get(userNames[2]).size()); Assert.assertEquals("create", pps.getUserPrivileges().get(userNames[2]).get(0).getPrivilege()); Assert.assertEquals(0, pps.getRolePrivilegesSize()); } private PrincipalPrivilegeSet getPPS(HiveObjectType objectType, String dbName, String tableName, String userName) throws InvalidObjectException, MetaException { switch (objectType) { case GLOBAL: return store.getUserPrivilegeSet(userName, null); case DATABASE: return store.getDBPrivilegeSet(dbName, userName, null); case TABLE: return store.getTablePrivilegeSet(dbName, tableName, userName, null); default: throw new RuntimeException("huh?"); } } private void checkRoleRemovedFromAllPrivileges(HiveObjectType objectType, String dbName, String tableName, String roleName) throws IOException, NoSuchObjectException, MetaException { List<PrivilegeGrantInfo> pgi = null; switch (objectType) { case GLOBAL: pgi = HBaseReadWrite.getInstance().getGlobalPrivs().getRolePrivileges().get(roleName); break; case DATABASE: pgi = store.getDatabase(dbName).getPrivileges().getRolePrivileges().get(roleName); break; case TABLE: pgi = store.getTable(dbName, tableName).getPrivileges().getRolePrivileges().get(roleName); break; default: Assert.fail(); } Assert.assertNull("Expected null for role " + roleName + " for type " + objectType.toString() + " with db " + dbName + " and table " + tableName, pgi); } @Test public void listDbGrants() throws Exception { String dbNames[] = new String[] {"ldbg_db1", "ldbg_db2"}; try { Database db = new Database(dbNames[0], "no description", "file:///tmp", emptyParameters); store.createDatabase(db); db = new Database(dbNames[1], "no description", "file:///tmp", emptyParameters); store.createDatabase(db); String[] roleNames = new String[]{"ldbg_role1", "ldbg_role2"}; String[] userNames = new String[]{"frodo", "sam"}; store.addRole(roleNames[0], "me"); store.addRole(roleNames[1], "me"); int now = (int)(System.currentTimeMillis() / 1000); Role role1 = store.getRole(roleNames[0]); Role role2 = store.getRole(roleNames[1]); store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false); store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true); store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false); List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>(); HiveObjectRef hiveObjRef = new HiveObjectRef(HiveObjectType.DATABASE, dbNames[0], null, null, null); PrivilegeGrantInfo grantInfo = new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false); HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER, grantInfo); privileges.add(hop); grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true); hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo); privileges.add(hop); PrivilegeBag pBag = new PrivilegeBag(privileges); store.grantPrivileges(pBag); List<HiveObjectPrivilege> hops = store.listPrincipalDBGrants(roleNames[0], PrincipalType.ROLE, dbNames[0]); Assert.assertEquals(1, hops.size()); Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType()); Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType()); Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege()); hops = store.listPrincipalDBGrants(userNames[0], PrincipalType.USER, dbNames[0]); Assert.assertEquals(1, hops.size()); Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType()); Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType()); Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege()); hops = store.listPrincipalDBGrants(roleNames[1], PrincipalType.ROLE, dbNames[0]); Assert.assertEquals(0, hops.size()); hops = store.listPrincipalDBGrants(userNames[1], PrincipalType.USER, dbNames[0]); Assert.assertEquals(0, hops.size()); hops = store.listPrincipalDBGrants(roleNames[0], PrincipalType.ROLE, dbNames[1]); Assert.assertEquals(0, hops.size()); hops = store.listPrincipalDBGrants(userNames[0], PrincipalType.USER, dbNames[1]); Assert.assertEquals(0, hops.size()); hops = store.listDBGrantsAll(dbNames[0]); Assert.assertEquals(2, hops.size()); boolean sawUser = false, sawRole = false; for (HiveObjectPrivilege h : hops) { if (h.getPrincipalName().equals(userNames[0])) { Assert.assertEquals(PrincipalType.USER, h.getPrincipalType()); Assert.assertEquals(HiveObjectType.DATABASE, h.getHiveObject().getObjectType()); Assert.assertEquals("read", h.getGrantInfo().getPrivilege()); sawUser = true; } else if (h.getPrincipalName().equals(roleNames[0])) { Assert.assertEquals(PrincipalType.ROLE, h.getPrincipalType()); Assert.assertEquals(HiveObjectType.DATABASE, h.getHiveObject().getObjectType()); Assert.assertEquals("write", h.getGrantInfo().getPrivilege()); sawRole = true; } } Assert.assertTrue(sawUser && sawRole); hops = store.listPrincipalDBGrantsAll(roleNames[0], PrincipalType.ROLE); Assert.assertEquals(1, hops.size()); Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType()); Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType()); Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege()); hops = store.listPrincipalDBGrantsAll(userNames[0], PrincipalType.USER); Assert.assertEquals(1, hops.size()); Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType()); Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType()); Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege()); hops = store.listPrincipalDBGrantsAll(roleNames[1], PrincipalType.ROLE); Assert.assertEquals(0, hops.size()); hops = store.listPrincipalDBGrantsAll(userNames[1], PrincipalType.USER); Assert.assertEquals(0, hops.size()); } finally { store.dropDatabase(dbNames[0]); store.dropDatabase(dbNames[1]); } } @Test public void listGlobalGrants() throws Exception { String[] roleNames = new String[]{"lgg_role1", "lgg_role2"}; String[] userNames = new String[]{"merry", "pippen"}; store.addRole(roleNames[0], "me"); store.addRole(roleNames[1], "me"); int now = (int)(System.currentTimeMillis() / 1000); Role role1 = store.getRole(roleNames[0]); Role role2 = store.getRole(roleNames[1]); store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false); store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true); store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false); List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>(); HiveObjectRef hiveObjRef = new HiveObjectRef(HiveObjectType.GLOBAL, null, null, null, null); PrivilegeGrantInfo grantInfo = new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false); HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER, grantInfo); privileges.add(hop); grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true); hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo); privileges.add(hop); PrivilegeBag pBag = new PrivilegeBag(privileges); store.grantPrivileges(pBag); List<HiveObjectPrivilege> hops = store.listPrincipalGlobalGrants(roleNames[0], PrincipalType.ROLE); Assert.assertEquals(1, hops.size()); Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType()); Assert.assertEquals(HiveObjectType.GLOBAL, hops.get(0).getHiveObject().getObjectType()); Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege()); hops = store.listPrincipalGlobalGrants(userNames[0], PrincipalType.USER); Assert.assertEquals(1, hops.size()); Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType()); Assert.assertEquals(HiveObjectType.GLOBAL, hops.get(0).getHiveObject().getObjectType()); Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege()); hops = store.listPrincipalGlobalGrants(roleNames[1], PrincipalType.ROLE); Assert.assertEquals(0, hops.size()); hops = store.listPrincipalGlobalGrants(userNames[1], PrincipalType.USER); Assert.assertEquals(0, hops.size()); hops = store.listGlobalGrantsAll(); Assert.assertEquals(2, hops.size()); boolean sawUser = false, sawRole = false; for (HiveObjectPrivilege h : hops) { if (h.getPrincipalName().equals(userNames[0])) { Assert.assertEquals(PrincipalType.USER, h.getPrincipalType()); Assert.assertEquals(HiveObjectType.GLOBAL, h.getHiveObject().getObjectType()); Assert.assertEquals("read", h.getGrantInfo().getPrivilege()); sawUser = true; } else if (h.getPrincipalName().equals(roleNames[0])) { Assert.assertEquals(PrincipalType.ROLE, h.getPrincipalType()); Assert.assertEquals(HiveObjectType.GLOBAL, h.getHiveObject().getObjectType()); Assert.assertEquals("write", h.getGrantInfo().getPrivilege()); sawRole = true; } } Assert.assertTrue(sawUser && sawRole); } @Test public void listTableGrants() throws Exception { String dbName = "ltg_db"; String[] tableNames = new String[] {"ltg_t1", "ltg_t2"}; try { Database db = new Database(dbName, "no description", "file:///tmp", emptyParameters); store.createDatabase(db); int startTime = (int)(System.currentTimeMillis() / 1000); List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema("col1", "int", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); Table table = new Table(tableNames[0], dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null); store.createTable(table); table = new Table(tableNames[1], dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null); store.createTable(table); String[] roleNames = new String[]{"ltg_role1", "ltg_role2"}; String[] userNames = new String[]{"gandalf", "radagast"}; store.addRole(roleNames[0], "me"); store.addRole(roleNames[1], "me"); int now = (int)(System.currentTimeMillis() / 1000); Role role1 = store.getRole(roleNames[0]); Role role2 = store.getRole(roleNames[1]); store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false); store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true); store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false); List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>(); HiveObjectRef hiveObjRef = new HiveObjectRef(HiveObjectType.TABLE, dbName, tableNames[0], null, null); PrivilegeGrantInfo grantInfo = new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false); HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER, grantInfo); privileges.add(hop); grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true); hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo); privileges.add(hop); PrivilegeBag pBag = new PrivilegeBag(privileges); store.grantPrivileges(pBag); List<HiveObjectPrivilege> hops = store.listAllTableGrants(roleNames[0], PrincipalType.ROLE, dbName, tableNames[0]); Assert.assertEquals(1, hops.size()); Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType()); Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType()); Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege()); hops = store.listAllTableGrants(userNames[0], PrincipalType.USER, dbName, tableNames[0]); Assert.assertEquals(1, hops.size()); Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType()); Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType()); Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege()); hops = store.listAllTableGrants(roleNames[1], PrincipalType.ROLE, dbName, tableNames[0]); Assert.assertEquals(0, hops.size()); hops = store.listAllTableGrants(userNames[1], PrincipalType.USER, dbName, tableNames[0]); Assert.assertEquals(0, hops.size()); hops = store.listAllTableGrants(roleNames[0], PrincipalType.ROLE, dbName, tableNames[1]); Assert.assertEquals(0, hops.size()); hops = store.listAllTableGrants(userNames[0], PrincipalType.USER, dbName, tableNames[1]); Assert.assertEquals(0, hops.size()); hops = store.listTableGrantsAll(dbName, tableNames[0]); Assert.assertEquals(2, hops.size()); boolean sawUser = false, sawRole = false; for (HiveObjectPrivilege h : hops) { if (h.getPrincipalName().equals(userNames[0])) { Assert.assertEquals(PrincipalType.USER, h.getPrincipalType()); Assert.assertEquals(HiveObjectType.TABLE, h.getHiveObject().getObjectType()); Assert.assertEquals("read", h.getGrantInfo().getPrivilege()); sawUser = true; } else if (h.getPrincipalName().equals(roleNames[0])) { Assert.assertEquals(PrincipalType.ROLE, h.getPrincipalType()); Assert.assertEquals(HiveObjectType.TABLE, h.getHiveObject().getObjectType()); Assert.assertEquals("write", h.getGrantInfo().getPrivilege()); sawRole = true; } } Assert.assertTrue(sawUser && sawRole); hops = store.listPrincipalTableGrantsAll(roleNames[0], PrincipalType.ROLE); Assert.assertEquals(1, hops.size()); Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType()); Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType()); Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege()); hops = store.listPrincipalTableGrantsAll(userNames[0], PrincipalType.USER); Assert.assertEquals(1, hops.size()); Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType()); Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType()); Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege()); hops = store.listPrincipalDBGrantsAll(roleNames[1], PrincipalType.ROLE); Assert.assertEquals(0, hops.size()); hops = store.listPrincipalDBGrantsAll(userNames[1], PrincipalType.USER); Assert.assertEquals(0, hops.size()); } finally { store.dropTable(dbName, tableNames[0]); store.dropTable(dbName, tableNames[1]); store.dropDatabase(dbName); } } @Test public void tableStatistics() throws Exception { long now = System.currentTimeMillis(); String dbname = "default"; String tableName = "statstable"; String boolcol = "boolcol"; String longcol = "longcol"; String doublecol = "doublecol"; String stringcol = "stringcol"; String binarycol = "bincol"; String decimalcol = "deccol"; long trues = 37; long falses = 12; long booleanNulls = 2; long longHigh = 120938479124L; long longLow = -12341243213412124L; long longNulls = 23; long longDVs = 213L; double doubleHigh = 123423.23423; double doubleLow = 0.00001234233; long doubleNulls = 92; long doubleDVs = 1234123421L; long strMaxLen = 1234; double strAvgLen = 32.3; long strNulls = 987; long strDVs = 906; long binMaxLen = 123412987L; double binAvgLen = 76.98; long binNulls = 976998797L; Decimal decHigh = new Decimal(); decHigh.setScale((short)3); decHigh.setUnscaled("3876".getBytes()); // I have no clue how this is translated, but it // doesn't matter Decimal decLow = new Decimal(); decLow.setScale((short)3); decLow.setUnscaled("38".getBytes()); long decNulls = 13; long decDVs = 923947293L; List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema(boolcol, "boolean", "nocomment")); cols.add(new FieldSchema(longcol, "long", "nocomment")); cols.add(new FieldSchema(doublecol, "double", "nocomment")); cols.add(new FieldSchema(stringcol, "varchar(32)", "nocomment")); cols.add(new FieldSchema(binarycol, "binary", "nocomment")); cols.add(new FieldSchema(decimalcol, "decimal(5, 3)", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); Table table = new Table(tableName, dbname, "me", (int)now / 1000, (int)now / 1000, 0, sd, null, emptyParameters, null, null, null); store.createTable(table); ColumnStatistics stats = new ColumnStatistics(); ColumnStatisticsDesc desc = new ColumnStatisticsDesc(); desc.setLastAnalyzed(now); desc.setDbName(dbname); desc.setTableName(tableName); desc.setIsTblLevel(true); stats.setStatsDesc(desc); // Do one column of each type ColumnStatisticsObj obj = new ColumnStatisticsObj(); obj.setColName(boolcol); obj.setColType("boolean"); ColumnStatisticsData data = new ColumnStatisticsData(); BooleanColumnStatsData boolData = new BooleanColumnStatsData(); boolData.setNumTrues(trues); boolData.setNumFalses(falses); boolData.setNumNulls(booleanNulls); data.setBooleanStats(boolData); obj.setStatsData(data); stats.addToStatsObj(obj); obj = new ColumnStatisticsObj(); obj.setColName(longcol); obj.setColType("long"); data = new ColumnStatisticsData(); LongColumnStatsData longData = new LongColumnStatsData(); longData.setHighValue(longHigh); longData.setLowValue(longLow); longData.setNumNulls(longNulls); longData.setNumDVs(longDVs); data.setLongStats(longData); obj.setStatsData(data); stats.addToStatsObj(obj); obj = new ColumnStatisticsObj(); obj.setColName(doublecol); obj.setColType("double"); data = new ColumnStatisticsData(); DoubleColumnStatsData doubleData = new DoubleColumnStatsData(); doubleData.setHighValue(doubleHigh); doubleData.setLowValue(doubleLow); doubleData.setNumNulls(doubleNulls); doubleData.setNumDVs(doubleDVs); data.setDoubleStats(doubleData); obj.setStatsData(data); stats.addToStatsObj(obj); store.updateTableColumnStatistics(stats); stats = store.getTableColumnStatistics(dbname, tableName, Arrays.asList(boolcol, longcol, doublecol)); // We'll check all of the individual values later. Assert.assertEquals(3, stats.getStatsObjSize()); // check that we can fetch just some of the columns stats = store.getTableColumnStatistics(dbname, tableName, Arrays.asList(boolcol)); Assert.assertEquals(1, stats.getStatsObjSize()); stats = new ColumnStatistics(); stats.setStatsDesc(desc); obj = new ColumnStatisticsObj(); obj.setColName(stringcol); obj.setColType("string"); data = new ColumnStatisticsData(); StringColumnStatsData strData = new StringColumnStatsData(); strData.setMaxColLen(strMaxLen); strData.setAvgColLen(strAvgLen); strData.setNumNulls(strNulls); strData.setNumDVs(strDVs); data.setStringStats(strData); obj.setStatsData(data); stats.addToStatsObj(obj); obj = new ColumnStatisticsObj(); obj.setColName(binarycol); obj.setColType("binary"); data = new ColumnStatisticsData(); BinaryColumnStatsData binData = new BinaryColumnStatsData(); binData.setMaxColLen(binMaxLen); binData.setAvgColLen(binAvgLen); binData.setNumNulls(binNulls); data.setBinaryStats(binData); obj.setStatsData(data); stats.addToStatsObj(obj); obj = new ColumnStatisticsObj(); obj.setColName(decimalcol); obj.setColType("decimal(5,3)"); data = new ColumnStatisticsData(); DecimalColumnStatsData decData = new DecimalColumnStatsData(); LOG.debug("Setting decimal high value to " + decHigh.getScale() + " <" + new String(decHigh.getUnscaled()) + ">"); decData.setHighValue(decHigh); decData.setLowValue(decLow); decData.setNumNulls(decNulls); decData.setNumDVs(decDVs); data.setDecimalStats(decData); obj.setStatsData(data); stats.addToStatsObj(obj); store.updateTableColumnStatistics(stats); stats = store.getTableColumnStatistics(dbname, tableName, Arrays.asList(boolcol, longcol, doublecol, stringcol, binarycol, decimalcol)); Assert.assertEquals(now, stats.getStatsDesc().getLastAnalyzed()); Assert.assertEquals(dbname, stats.getStatsDesc().getDbName()); Assert.assertEquals(tableName, stats.getStatsDesc().getTableName()); Assert.assertTrue(stats.getStatsDesc().isIsTblLevel()); Assert.assertEquals(6, stats.getStatsObjSize()); ColumnStatisticsData colData = stats.getStatsObj().get(0).getStatsData(); Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS, colData.getSetField()); boolData = colData.getBooleanStats(); Assert.assertEquals(trues, boolData.getNumTrues()); Assert.assertEquals(falses, boolData.getNumFalses()); Assert.assertEquals(booleanNulls, boolData.getNumNulls()); colData = stats.getStatsObj().get(1).getStatsData(); Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, colData.getSetField()); longData = colData.getLongStats(); Assert.assertEquals(longHigh, longData.getHighValue()); Assert.assertEquals(longLow, longData.getLowValue()); Assert.assertEquals(longNulls, longData.getNumNulls()); Assert.assertEquals(longDVs, longData.getNumDVs()); colData = stats.getStatsObj().get(2).getStatsData(); Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, colData.getSetField()); doubleData = colData.getDoubleStats(); Assert.assertEquals(doubleHigh, doubleData.getHighValue(), 0.01); Assert.assertEquals(doubleLow, doubleData.getLowValue(), 0.01); Assert.assertEquals(doubleNulls, doubleData.getNumNulls()); Assert.assertEquals(doubleDVs, doubleData.getNumDVs()); colData = stats.getStatsObj().get(3).getStatsData(); Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, colData.getSetField()); strData = colData.getStringStats(); Assert.assertEquals(strMaxLen, strData.getMaxColLen()); Assert.assertEquals(strAvgLen, strData.getAvgColLen(), 0.01); Assert.assertEquals(strNulls, strData.getNumNulls()); Assert.assertEquals(strDVs, strData.getNumDVs()); colData = stats.getStatsObj().get(4).getStatsData(); Assert.assertEquals(ColumnStatisticsData._Fields.BINARY_STATS, colData.getSetField()); binData = colData.getBinaryStats(); Assert.assertEquals(binMaxLen, binData.getMaxColLen()); Assert.assertEquals(binAvgLen, binData.getAvgColLen(), 0.01); Assert.assertEquals(binNulls, binData.getNumNulls()); colData = stats.getStatsObj().get(5).getStatsData(); Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, colData.getSetField()); decData = colData.getDecimalStats(); Assert.assertEquals(decHigh, decData.getHighValue()); Assert.assertEquals(decLow, decData.getLowValue()); Assert.assertEquals(decNulls, decData.getNumNulls()); Assert.assertEquals(decDVs, decData.getNumDVs()); } @Test public void partitionStatistics() throws Exception { long now = System.currentTimeMillis(); String dbname = "default"; String tableName = "statspart"; String[] partNames = {"ds=today", "ds=yesterday"}; String[] partVals = {"today", "yesterday"}; String boolcol = "boolcol"; String longcol = "longcol"; String doublecol = "doublecol"; String stringcol = "stringcol"; String binarycol = "bincol"; String decimalcol = "deccol"; long trues = 37; long falses = 12; long booleanNulls = 2; long strMaxLen = 1234; double strAvgLen = 32.3; long strNulls = 987; long strDVs = 906; List<FieldSchema> cols = new ArrayList<FieldSchema>(); cols.add(new FieldSchema(boolcol, "boolean", "nocomment")); cols.add(new FieldSchema(longcol, "long", "nocomment")); cols.add(new FieldSchema(doublecol, "double", "nocomment")); cols.add(new FieldSchema(stringcol, "varchar(32)", "nocomment")); cols.add(new FieldSchema(binarycol, "binary", "nocomment")); cols.add(new FieldSchema(decimalcol, "decimal(5, 3)", "nocomment")); SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("ds", "string", "")); Table table = new Table(tableName, dbname, "me", (int)now / 1000, (int)now / 1000, 0, sd, partCols, emptyParameters, null, null, null); store.createTable(table); for (String partVal : partVals) { Partition part = new Partition(Arrays.asList(partVal), dbname, tableName, (int) now / 1000, (int) now / 1000, sd, emptyParameters); store.addPartition(part); } for (int i = 0; i < partNames.length; i++) { ColumnStatistics stats = new ColumnStatistics(); ColumnStatisticsDesc desc = new ColumnStatisticsDesc(); desc.setLastAnalyzed(now); desc.setDbName(dbname); desc.setTableName(tableName); desc.setIsTblLevel(false); desc.setPartName(partNames[i]); stats.setStatsDesc(desc); ColumnStatisticsObj obj = new ColumnStatisticsObj(); obj.setColName(boolcol); obj.setColType("boolean"); ColumnStatisticsData data = new ColumnStatisticsData(); BooleanColumnStatsData boolData = new BooleanColumnStatsData(); boolData.setNumTrues(trues); boolData.setNumFalses(falses); boolData.setNumNulls(booleanNulls); data.setBooleanStats(boolData); obj.setStatsData(data); stats.addToStatsObj(obj); store.updatePartitionColumnStatistics(stats, Arrays.asList(partVals[i])); } List<ColumnStatistics> statsList = store.getPartitionColumnStatistics(dbname, tableName, Arrays.asList(partNames), Arrays.asList(boolcol)); Assert.assertEquals(2, statsList.size()); for (int i = 0; i < partNames.length; i++) { Assert.assertEquals(1, statsList.get(i).getStatsObjSize()); } for (int i = 0; i < partNames.length; i++) { ColumnStatistics stats = new ColumnStatistics(); ColumnStatisticsDesc desc = new ColumnStatisticsDesc(); desc.setLastAnalyzed(now); desc.setDbName(dbname); desc.setTableName(tableName); desc.setIsTblLevel(false); desc.setPartName(partNames[i]); stats.setStatsDesc(desc); ColumnStatisticsObj obj = new ColumnStatisticsObj(); obj.setColName(stringcol); obj.setColType("string"); ColumnStatisticsData data = new ColumnStatisticsData(); StringColumnStatsData strData = new StringColumnStatsData(); strData.setMaxColLen(strMaxLen); strData.setAvgColLen(strAvgLen); strData.setNumNulls(strNulls); strData.setNumDVs(strDVs); data.setStringStats(strData); obj.setStatsData(data); stats.addToStatsObj(obj); store.updatePartitionColumnStatistics(stats, Arrays.asList(partVals[i])); } // Make sure when we ask for one we only get one statsList = store.getPartitionColumnStatistics(dbname, tableName, Arrays.asList(partNames), Arrays.asList(boolcol)); Assert.assertEquals(2, statsList.size()); for (int i = 0; i < partNames.length; i++) { Assert.assertEquals(1, statsList.get(i).getStatsObjSize()); } statsList = store.getPartitionColumnStatistics(dbname, tableName, Arrays.asList(partNames), Arrays.asList(boolcol, stringcol)); Assert.assertEquals(2, statsList.size()); for (int i = 0; i < partNames.length; i++) { Assert.assertEquals(2, statsList.get(i).getStatsObjSize()); // Just check one piece of the data, I don't need to check it all again Assert.assertEquals(booleanNulls, statsList.get(i).getStatsObj().get(0).getStatsData().getBooleanStats().getNumNulls()); Assert.assertEquals(strDVs, statsList.get(i).getStatsObj().get(1).getStatsData().getStringStats().getNumDVs()); } } @Test public void delegationToken() throws Exception { store.addToken("abc", "def"); store.addToken("ghi", "jkl"); Assert.assertEquals("def", store.getToken("abc")); Assert.assertEquals("jkl", store.getToken("ghi")); Assert.assertNull(store.getToken("wabawaba")); String[] allToks = store.getAllTokenIdentifiers().toArray(new String[2]); Arrays.sort(allToks); Assert.assertArrayEquals(new String[]{"abc", "ghi"}, allToks); store.removeToken("abc"); store.removeToken("wabawaba"); Assert.assertNull(store.getToken("abc")); Assert.assertEquals("jkl", store.getToken("ghi")); allToks = store.getAllTokenIdentifiers().toArray(new String[1]); Assert.assertArrayEquals(new String[]{"ghi"}, allToks); } @Test public void masterKey() throws Exception { Assert.assertEquals(0, store.addMasterKey("k1")); Assert.assertEquals(1, store.addMasterKey("k2")); String[] keys = store.getMasterKeys(); Arrays.sort(keys); Assert.assertArrayEquals(new String[]{"k1", "k2"}, keys); store.updateMasterKey(0, "k3"); keys = store.getMasterKeys(); Arrays.sort(keys); Assert.assertArrayEquals(new String[]{"k2", "k3"}, keys); store.removeMasterKey(1); keys = store.getMasterKeys(); Assert.assertArrayEquals(new String[]{"k3"}, keys); thrown.expect(NoSuchObjectException.class); store.updateMasterKey(72, "whatever"); } }