/**
* Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.linkedin.pinot.integration.tests;
import com.google.common.util.concurrent.Uninterruptibles;
import com.linkedin.pinot.common.data.Schema;
import java.io.File;
import java.util.concurrent.TimeUnit;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
/**
* Test that changes the schema at runtime during an integration test.
*/
public class RealtimeSchemaChangeIntegrationTest extends RealtimeClusterIntegrationTest {
private static final Logger LOGGER = LoggerFactory.getLogger(RealtimeSchemaChangeIntegrationTest.class);
private File getPartialSchemaFile() {
return new File(OfflineClusterIntegrationTest.class.getClassLoader()
.getResource("On_Time_On_Time_Performance_2014_100k_subset_nonulls_missing_columns.schema").getFile());
}
@Override
protected void setUpTable(String tableName, String timeColumnName, String timeColumnType, String kafkaZkUrl,
String kafkaTopic, File schemaFile, File avroFile) throws Exception {
Schema schema = Schema.fromFile(schemaFile);
addSchema(getPartialSchemaFile(), schema.getSchemaName());
// Create the table with a different schema than the final one
addRealtimeTable(tableName, timeColumnName, timeColumnType, 900, "Days", kafkaZkUrl, kafkaTopic,
schema.getSchemaName(), null, null, avroFile, ROW_COUNT_FOR_REALTIME_SEGMENT_FLUSH, "Carrier");
// Sleep for a little bit to get some events in the realtime table
Uninterruptibles.sleepUninterruptibly(15L, TimeUnit.SECONDS);
// Change the schema to the final one
updateSchema(schemaFile, schema.getSchemaName());
}
@Test(enabled = false)
public void testForColumnPartialPresence() throws Exception {
// Count the number of rows
JSONObject response = postQuery("select count(*) from 'mytable'");
JSONArray aggregationResultsArray = response.getJSONArray("aggregationResults");
JSONObject firstAggregationResult = aggregationResultsArray.getJSONObject(0);
String pinotValue = firstAggregationResult.getString("value");
int totalRecordCount = Integer.parseInt(pinotValue);
// Count the number of rows where the new column exists
response = postQuery("select count(*) from 'mytable' where ArrDel15 >= -10000");
aggregationResultsArray = response.getJSONArray("aggregationResults");
firstAggregationResult = aggregationResultsArray.getJSONObject(0);
pinotValue = firstAggregationResult.getString("value");
int rowsWithNewColumn = Integer.parseInt(pinotValue);
// Both numbers shouldn't match, since the schema was changed
LOGGER.info("totalRecordCount = " + totalRecordCount);
LOGGER.info("rowsWithNewColumn = " + rowsWithNewColumn);
Assert.assertTrue(totalRecordCount != rowsWithNewColumn,
"The number of rows consumed with the new schema is equal to the total number of rows");
}
@Override
@Test(enabled = false)
public void testHardcodedQueries()
throws Exception {
// Ignored.
}
@Override
@Test(enabled = false)
public void testHardcodedQuerySet()
throws Exception {
// Ignored.
}
@Override
@Test(enabled = false)
public void testGeneratedQueriesWithoutMultiValues()
throws Exception {
// Ignored.
}
@Override
@Test(enabled = false)
public void testGeneratedQueriesWithMultiValues()
throws Exception {
// Ignored.
}
}