/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.facebook.presto.kafka; import com.facebook.presto.Session; import com.facebook.presto.kafka.util.EmbeddedKafka; import com.facebook.presto.kafka.util.TestUtils; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.type.BigintType; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.tests.StandaloneQueryRunner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import kafka.producer.KeyedMessage; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.util.Properties; import java.util.UUID; import static com.facebook.presto.kafka.util.EmbeddedKafka.CloseableProducer; import static com.facebook.presto.kafka.util.TestUtils.createEmptyTopicDescription; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.testing.assertions.Assert.assertEquals; @Test(singleThreaded = true) public class TestManySegments { private static final Session SESSION = testSessionBuilder() .setCatalog("kafka") .setSchema("default") .build(); private EmbeddedKafka embeddedKafka; private String topicName; private StandaloneQueryRunner queryRunner; @BeforeClass public void startKafka() throws Exception { embeddedKafka = EmbeddedKafka.createEmbeddedKafka(); embeddedKafka.start(); topicName = "test_" + UUID.randomUUID().toString().replaceAll("-", "_"); Properties topicProperties = new Properties(); topicProperties.setProperty("segment.bytes", "1048576"); embeddedKafka.createTopics(1, 1, topicProperties, topicName); try (CloseableProducer<Long, Object> producer = embeddedKafka.createProducer()) { int jMax = 10_000; int iMax = 100_000 / jMax; for (long i = 0; i < iMax; i++) { ImmutableList.Builder<KeyedMessage<Long, Object>> builder = ImmutableList.builder(); for (long j = 0; j < jMax; j++) { builder.add(new KeyedMessage<Long, Object>(topicName, i, ImmutableMap.of("id", Long.toString(i * iMax + j), "value", UUID.randomUUID().toString()))); } producer.send(builder.build()); } } } @AfterClass(alwaysRun = true) public void stopKafka() throws Exception { embeddedKafka.close(); } @BeforeMethod public void spinUp() throws Exception { this.queryRunner = new StandaloneQueryRunner(SESSION); TestUtils.installKafkaPlugin(embeddedKafka, queryRunner, ImmutableMap.<SchemaTableName, KafkaTopicDescription>builder() .put(createEmptyTopicDescription(topicName, new SchemaTableName("default", topicName))) .build()); } @AfterMethod(alwaysRun = true) public void tearDown() throws Exception { queryRunner.close(); } @Test public void testManySegments() throws Exception { MaterializedResult result = queryRunner.execute("SELECT count(_message) from " + topicName); MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) .row(100000L) .build(); assertEquals(result, expected); } }