/**
* Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.linkedin.pinot.integration.tests;
import com.linkedin.pinot.controller.helix.ControllerTest;
import com.linkedin.pinot.segments.v1.creator.SegmentTestUtils;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.math.IntRange;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.json.JSONObject;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import com.linkedin.pinot.common.data.FieldSpec;
import com.linkedin.pinot.common.data.FieldSpec.DataType;
import com.linkedin.pinot.common.data.FieldSpec.FieldType;
import com.linkedin.pinot.core.data.readers.FileFormat;
import com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver;
import com.linkedin.pinot.core.segment.creator.impl.SegmentCreationDriverFactory;
import com.linkedin.pinot.tools.data.generator.DataGenerator;
import com.linkedin.pinot.tools.data.generator.DataGeneratorSpec;
/**
* Sep 14, 2014
*/
public class FileBasedSentineTest extends ControllerTest {
private static final Logger LOGGER = LoggerFactory.getLogger(FileBasedSentineTest.class);
private static URL url;
private static final String AVRO_FILE_PATH = "/tmp/avroFiles";
FileBasedServerBrokerStarters starter;
DataGenerator generator;
private static File avroDataDir = new File(AVRO_FILE_PATH);
@BeforeClass
public void setup() throws Exception {
url = new URL("http://localhost:" + FileBasedServerBrokerStarters.BROKER_CLIENT_PORT + "/query");
// lets generate data
final String[] columns = { "dimention1", "dimention2", "dimention3", "dimention4", "metric1", "daysSinceEpoch" };
final Map<String, DataType> dataTypes = new HashMap<String, FieldSpec.DataType>();
final Map<String, FieldType> fieldTypes = new HashMap<String, FieldType>();
final Map<String, TimeUnit> timeUnits = new HashMap<String, TimeUnit>();
final Map<String, Integer> cardinality = new HashMap<String, Integer>();
// Crate empty range map as the signature of DataGeneratorSpec has changed, and this test does not
// use metric/time as fieldType.
final Map<String, IntRange> range = new HashMap<String, IntRange>();
for (final String col : columns) {
if (col.equals("dimention1")) {
dataTypes.put(col, DataType.STRING);
cardinality.put(col, 1000);
} else {
dataTypes.put(col, DataType.INT);
cardinality.put(col, 1000);
}
fieldTypes.put(col, FieldType.DIMENSION);
}
if (avroDataDir.exists()) {
FileUtils.deleteDirectory(avroDataDir);
}
final DataGeneratorSpec spec =
new DataGeneratorSpec(Arrays.asList(columns), cardinality, range, dataTypes, fieldTypes, timeUnits,
FileFormat.AVRO, avroDataDir.getAbsolutePath(), true);
generator = new DataGenerator();
generator.init(spec);
generator.generate(100000L, 2);
// lets make segments now
final File bootstrapDir = new File(FileBasedServerBrokerStarters.SERVER_BOOTSTRAP_DIR);
if (bootstrapDir.exists()) {
FileUtils.deleteDirectory(bootstrapDir);
}
bootstrapDir.mkdir();
int counter = 0;
for (final File avro : avroDataDir.listFiles()) {
for (final String table : FileBasedServerBrokerStarters.TABLE_NAMES) {
final SegmentGeneratorConfig genConfig = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(avro,
new File(bootstrapDir, "segment-" + counter), "daysSinceEpoch", TimeUnit.DAYS, table);
final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
driver.init(genConfig);
driver.build();
counter++;
}
}
// lets start the server and the broker now
starter = new FileBasedServerBrokerStarters();
starter.startAll();
// pick some values from here if you need to use it for running filter queries
final JSONObject selectionRequestResponse = postQuery("select * from 'table1' limit 100",
"http://localhost:" + FileBasedServerBrokerStarters.BROKER_CLIENT_PORT);
// System.out.println(selectionRequestResponse.toString(1));
}
@AfterClass
public void tearDown() throws IOException {
starter.stopAll();
FileUtils.deleteDirectory(new File(FileBasedServerBrokerStarters.SERVER_BOOTSTRAP_DIR));
FileUtils.deleteDirectory(avroDataDir);
FileUtils.deleteDirectory(new File(FileBasedServerBrokerStarters.SERVER_INDEX_DIR));
}
@Test
public void test1() {
}
}