CubeReducerTest.java example

Explorer
Kylin-master
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/

package org.apache.kylin.engine.mr.steps;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.File;
import java.lang.reflect.Field;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.apache.hadoop.mrunit.types.Pair;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

/**
 */
@SuppressWarnings("rawtypes")
public class CubeReducerTest extends LocalFileMetadataTestCase {

    ReduceDriver<Text, Text, Text, Text> reduceDriver;

    @Before
    public void setUp() throws Exception {
        createTestMetadata();

        // hack for distributed cache
        FileUtils.deleteDirectory(new File("../job/meta"));
        FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta"));

        CuboidReducer reducer = new CuboidReducer();
        reduceDriver = ReduceDriver.newReduceDriver(reducer);
    }

    @After
    public void after() throws Exception {
        cleanupTestMetadata();
        FileUtils.deleteDirectory(new File("../job/meta"));
    }

    @Test
    public void testReducer() throws Exception {

        reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");

        CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
        BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());

        Text key1 = new Text("72010ustech");
        List<Text> values1 = new ArrayList<Text>();
        values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100));
        values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200));
        values1.add(newValueText(codec, "10", "10", "10", 1, 300));

        Text key2 = new Text("1tech");
        List<Text> values2 = new ArrayList<Text>();
        values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500));
        values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000));

        Text key3 = new Text("0");
        List<Text> values3 = new ArrayList<Text>();
        values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0));

        reduceDriver.withInput(key1, values1);
        reduceDriver.withInput(key2, values2);
        reduceDriver.withInput(key3, values3);

        List<Pair<Text, Text>> result = reduceDriver.run();

        Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3, 600));
        Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2, 1500));
        Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 0, 0));

        assertEquals(3, result.size());

        assertTrue(result.contains(p1));
        assertTrue(result.contains(p2));
        assertTrue(result.contains(p3));
    }

    @Test
    public void testReducerOnlyAggrInBaseCuboid() throws Exception {
        reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
        reduceDriver.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 1);

        CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
        MeasureDesc measureDesc = cubeDesc.getMeasures().get(0);
        FunctionDesc functionDesc = measureDesc.getFunction();
        Field field = FunctionDesc.class.getDeclaredField("measureType");
        field.setAccessible(true);
        MeasureType origMeasureType = functionDesc.getMeasureType();
        field.set(functionDesc, new MockUpMeasureType(origMeasureType));

        BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());

        Text key1 = new Text("72010ustech");
        List<Text> values1 = new ArrayList<Text>();
        values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100));
        values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200));
        values1.add(newValueText(codec, "10", "10", "10", 1, 300));

        Text key2 = new Text("1tech");
        List<Text> values2 = new ArrayList<Text>();
        values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500));
        values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000));

        Text key3 = new Text("0");
        List<Text> values3 = new ArrayList<Text>();
        values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0));

        reduceDriver.withInput(key1, values1);
        reduceDriver.withInput(key2, values2);
        reduceDriver.withInput(key3, values3);

        List<Pair<Text, Text>> result = reduceDriver.run();

        Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "0", "10", "20.34", 3, 600));
        Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "0", "15.09", "20.34", 2, 1500));
        Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "0", "146.52", "146.52", 0, 0));

        assertEquals(3, result.size());

        assertTrue(result.contains(p1));
        assertTrue(result.contains(p2));
        assertTrue(result.contains(p3));
    }

    private Text newValueText(BufferedMeasureCodec codec, String sum, String min, String max, int count, int item_count) {
        Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new Long(count), new Long(item_count) };

        ByteBuffer buf = codec.encode(values);

        Text t = new Text();
        t.set(buf.array(), 0, buf.position());
        return t;
    }

    class MockUpMeasureType extends MeasureType {
        MeasureType origMeasureType;

        public MockUpMeasureType(MeasureType origMeasureType) {
            this.origMeasureType = origMeasureType;
        }

        @Override
        public boolean onlyAggrInBaseCuboid() {
            return true;
        }

        @Override
        public MeasureIngester newIngester() {
            return origMeasureType.newIngester();
        }

        @Override
        public MeasureAggregator newAggregator() {
            return origMeasureType.newAggregator();
        }

        @Override
        public boolean needRewrite() {
            return origMeasureType.needRewrite();
        }

        @Override
        public Map<String, Class<?>> getRewriteCalciteAggrFunctions() {
            return origMeasureType.getRewriteCalciteAggrFunctions();
        }
    }
}