/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hive.ql.index; import com.google.common.collect.ImmutableSet; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.hadoop.hive.ql.io.HiveInputFormat.HiveInputSplit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; public final class SplitFilterTestCase { public static final long DEFAULT_SPLIT_SIZE = 1024 * 1024; public static final long SMALL_SPLIT_SIZE = 500; private final Set<HiveInputSplit> allSplits; private final Set<HiveInputSplit> selectedSplits; private final Set<HiveInputSplit> expectedSplits; private final long maxInputSize; private SplitFilterTestCase(Iterable<HiveInputSplit> allSplits, Iterable<HiveInputSplit> selectedSplits, Iterable<HiveInputSplit> expectedSplits, long maxInputSize) { this.allSplits = ImmutableSet.copyOf(allSplits); this.selectedSplits = ImmutableSet.copyOf(selectedSplits); this.expectedSplits = ImmutableSet.copyOf(expectedSplits); this.maxInputSize = maxInputSize; } private HiveInputSplit[] toArray(Collection<HiveInputSplit> splits) { return splits.toArray(new HiveInputSplit[splits.size()]); } public void executeAndValidate() throws IOException { SplitFilter filter = new SplitFilter(new MockIndexResult(selectedSplits), maxInputSize); List<HiveInputSplit> actualSplits = filter.filter(toArray(allSplits)); assertSplits(expectedSplits, actualSplits); } private void assertSplits(Collection<HiveInputSplit> expectedSplits, Collection<HiveInputSplit> actualSplits) { SplitFilter.HiveInputSplitComparator hiveInputSplitComparator = new SplitFilter.HiveInputSplitComparator(); List<HiveInputSplit> sortedExpectedSplits = new ArrayList<>(expectedSplits); Collections.sort(sortedExpectedSplits, hiveInputSplitComparator); List<HiveInputSplit> sortedActualSplits = new ArrayList<>(actualSplits); Collections.sort(sortedActualSplits, hiveInputSplitComparator); assertEquals("Number of selected splits.", sortedExpectedSplits.size(), sortedActualSplits.size()); for (int i = 0; i < sortedExpectedSplits.size(); i++) { HiveInputSplit expectedSplit = sortedExpectedSplits.get(i); HiveInputSplit actualSplit = sortedActualSplits.get(i); String splitName = "Split #" + i; assertEquals(splitName + " path.", expectedSplit.getPath(), actualSplit.getPath()); assertEquals(splitName + " start.", expectedSplit.getStart(), actualSplit.getStart()); assertEquals(splitName + " length.", expectedSplit.getLength(), actualSplit.getLength()); } } public static MaxInputSizeStep builder() { return new SplitFilterTestCaseBuilder(); } public static interface MaxInputSizeStep extends InputFilesStep { InputFilesStep maxInputSize(long maxInputSize); } public static interface InputFilesStep { ExpectedSplitsStep inputFiles(MockInputFile... inputFiles); } public static interface ExpectedSplitsStep { BuildStep expectedSplits(HiveInputSplit... expectedSplits); } public static interface BuildStep { SplitFilterTestCase build(); } private static final class SplitFilterTestCaseBuilder implements MaxInputSizeStep, InputFilesStep, ExpectedSplitsStep, BuildStep { private long maxInputSize = Long.MAX_VALUE; private List<MockInputFile> inputFiles; private List<HiveInputSplit> expectedSplits; @Override public InputFilesStep maxInputSize(long maxInputSize) { this.maxInputSize = maxInputSize; return this; } @Override public ExpectedSplitsStep inputFiles(MockInputFile... inputFiles) { this.inputFiles = Arrays.asList(inputFiles); return this; } @Override public BuildStep expectedSplits(HiveInputSplit... expectedSplits) { this.expectedSplits = Arrays.asList(expectedSplits); return this; } @Override public SplitFilterTestCase build() { List<HiveInputSplit> allSplits = new ArrayList<>(); List<HiveInputSplit> selectedSplits = new ArrayList<>(); Set<String> seenPaths = new HashSet<String>(); for (MockInputFile inputFile : inputFiles) { if (seenPaths.add(inputFile.getPath())) { allSplits.addAll(inputFile.getSplits()); selectedSplits.addAll(inputFile.getSelectedSplits()); } else { fail(String.format("Cannot add 2 input files with the same path to a test case. " + "The duplicated path is '%s'.", inputFile.getPath())); } } return new SplitFilterTestCase(allSplits, selectedSplits, expectedSplits, maxInputSize); } } }