/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.common.config; import com.linkedin.pinot.common.data.StarTreeIndexSpec; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.json.JSONException; import org.json.JSONObject; import org.testng.Assert; import org.testng.annotations.Test; public class IndexingConfigTest { @Test public void testIgnoreUnknown() throws JSONException, IOException { JSONObject json = new JSONObject(); json.put("invertedIndexColumns", Arrays.asList("a", "b", "c")); json.put("sortedColumn", Arrays.asList("d", "e", "f")); json.put("loadMode", "MMAP"); json.put("keyThatIsUnknown", "randomValue"); ObjectMapper mapper = new ObjectMapper(); JsonNode jsonNode = mapper.readTree(json.toString()); IndexingConfig indexingConfig = mapper.readValue(jsonNode, IndexingConfig.class); Assert.assertEquals("MMAP", indexingConfig.getLoadMode()); List<String> invertedIndexColumns = indexingConfig.getInvertedIndexColumns(); Assert.assertEquals(3, invertedIndexColumns.size()); Assert.assertEquals("a", invertedIndexColumns.get(0)); Assert.assertEquals("b", invertedIndexColumns.get(1)); Assert.assertEquals("c", invertedIndexColumns.get(2)); List<String> sortedIndexColumns = indexingConfig.getSortedColumn(); Assert.assertEquals(3, sortedIndexColumns.size()); Assert.assertEquals("d", sortedIndexColumns.get(0)); Assert.assertEquals("e", sortedIndexColumns.get(1)); Assert.assertEquals("f", sortedIndexColumns.get(2)); } @Test public void testSegmentPartitionConfig() throws IOException { int numColumns = 5; Map<String, ColumnPartitionConfig> expectedColumnPartitionMap = new HashMap<>(5); for (int i = 0; i < numColumns; i++) { expectedColumnPartitionMap.put("column_" + i, new ColumnPartitionConfig("function_" + i, i + 1)); } SegmentPartitionConfig expectedPartitionConfig = new SegmentPartitionConfig(expectedColumnPartitionMap); IndexingConfig expectedIndexingConfig = new IndexingConfig(); expectedIndexingConfig.setSegmentPartitionConfig(expectedPartitionConfig); ObjectMapper mapper = new ObjectMapper(); String indexingConfigString = mapper.writeValueAsString(expectedIndexingConfig); IndexingConfig actualIndexingConfig = mapper.readValue(indexingConfigString, IndexingConfig.class); SegmentPartitionConfig actualPartitionConfig = actualIndexingConfig.getSegmentPartitionConfig(); Map<String, ColumnPartitionConfig> actualColumnPartitionMap = actualPartitionConfig.getColumnPartitionMap(); Assert.assertEquals(actualColumnPartitionMap.size(), expectedColumnPartitionMap.size()); for (String column : expectedColumnPartitionMap.keySet()) { Assert.assertEquals(actualPartitionConfig.getFunctionName(column), expectedPartitionConfig.getFunctionName(column)); Assert.assertEquals(actualPartitionConfig.getNumPartitions(column), expectedPartitionConfig.getNumPartitions(column)); } } /** * Unit test to check get and set of star tree index spec on IndexingConfig. * <ul> * <li> Creates a StarTreeIndexSpec and sets it into the IndexingConfig. </li> * <li> Indexing config is first serialized into a string, and then read back from string. </li> * <li> Test to ensure star tree index spec values are correct after serialization and de-serialization. </li> * </ul> * @throws IOException */ @Test public void testStarTreeSpec() throws IOException { Random random = new Random(System.nanoTime()); StarTreeIndexSpec expectedStarTreeSpec = new StarTreeIndexSpec(); List<String> expectedDimensionSplitOrder = Arrays.asList("col1", "col2", "col3"); expectedStarTreeSpec.setDimensionsSplitOrder(expectedDimensionSplitOrder); Integer expectedMaxLeafRecords = random.nextInt(); expectedStarTreeSpec.setMaxLeafRecords(expectedMaxLeafRecords); int expectedSkipMaterializationThreshold = random.nextInt(); expectedStarTreeSpec.setSkipMaterializationCardinalityThreshold(expectedSkipMaterializationThreshold); Set<String> expectedSkipMaterializationDimensions = new HashSet<>(Arrays.asList(new String[]{"col4", "col5"})); expectedStarTreeSpec.setSkipMaterializationForDimensions(expectedSkipMaterializationDimensions); Set<String> expectedSkipStarNodeCreationForDimension = new HashSet<>(Arrays.asList(new String[]{"col6", "col7"})); expectedStarTreeSpec.setSkipStarNodeCreationForDimensions(expectedSkipStarNodeCreationForDimension); IndexingConfig expectedIndexingConfig = new IndexingConfig(); expectedIndexingConfig.setStarTreeIndexSpec(expectedStarTreeSpec); ObjectMapper objectMapper = new ObjectMapper(); String indexingConfigString = objectMapper.writeValueAsString(expectedIndexingConfig); IndexingConfig actualIndexingConfig = objectMapper.readValue(indexingConfigString, IndexingConfig.class); StarTreeIndexSpec actualStarTreeSpec = actualIndexingConfig.getStarTreeIndexSpec(); Assert.assertEquals(actualStarTreeSpec.getDimensionsSplitOrder(), expectedDimensionSplitOrder); Assert.assertEquals(actualStarTreeSpec.getMaxLeafRecords(), expectedMaxLeafRecords); Assert.assertEquals(actualStarTreeSpec.getskipMaterializationCardinalityThreshold(), expectedSkipMaterializationThreshold); Assert.assertEquals(actualStarTreeSpec.getskipMaterializationForDimensions(), expectedSkipMaterializationDimensions); Assert.assertEquals(actualStarTreeSpec.getSkipStarNodeCreationForDimensions(), expectedSkipStarNodeCreationForDimension); } }