/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.kylin.engine.mr.steps; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.List; import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; import org.apache.hadoop.mrunit.types.Pair; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.kv.RowConstants; import org.apache.kylin.engine.mr.common.BatchConstants; import org.junit.After; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import com.google.common.base.Function; import com.google.common.collect.Collections2; public class NDCuboidMapperTest extends LocalFileMetadataTestCase { MapReduceDriver<Text, Text, Text, Text, Text, Text> mapReduceDriver; @Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("./meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta")); NDCuboidMapper mapper = new NDCuboidMapper(); CuboidReducer reducer = new CuboidReducer(); mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer); } @After public void after() throws Exception { cleanupTestMetadata(); FileUtils.deleteDirectory(new File("./meta")); } @Ignore @Test public void testMapReduceWithSlr() throws IOException { String cubeName = "test_kylin_cube_with_slr_1_new_segment"; String segmentID = "198va32a-a33e-4b69-83dd-0bb8b1f8c53b"; mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID); byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 55, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; byte[] value = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1, 1 }; Pair<Text, Text> input1 = new Pair<Text, Text>(new Text(key), new Text(value)); mapReduceDriver.addInput(input1); List<Pair<Text, Text>> result = mapReduceDriver.run(); assertEquals(4, result.size()); byte[] resultKey = { 0, 0, 0, 0, 0, 0, 0, 0, 1, 127, 0, -104, -106, -128, 55, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; byte[] resultValue = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1, 1 }; Pair<Text, Text> output1 = new Pair<Text, Text>(new Text(resultKey), new Text(resultValue)); //As we will truncate decimal(KYLIN-766), value will no longer equals to resultValue Collection<Text> keys = Collections2.transform(result, new Function<Pair<Text, Text>, Text>() { @Nullable @Override public Text apply(Pair<Text, Text> input) { return input.getFirst(); } }); assertTrue(keys.contains(output1.getFirst())); assertTrue(!result.contains(output1)); long[] keySet = new long[result.size()]; System.out.println(Bytes.toLong(new byte[] { 0, 0, 0, 0, 0, 0, 1, -1 })); for (int i = 0; i < result.size(); i++) { byte[] bytes = new byte[result.get(i).getFirst().getLength()]; System.arraycopy(result.get(i).getFirst().getBytes(), RowConstants.ROWKEY_SHARDID_LEN, bytes, 0, result.get(i).getFirst().getLength() - RowConstants.ROWKEY_SHARDID_LEN); System.out.println(Bytes.toLong(bytes)); keySet[i] = Bytes.toLong(bytes); } // refer to CuboidSchedulerTest.testGetSpanningCuboid() assertArrayEquals(new long[] { 383, 447, 503, 504 }, keySet); } }