/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.kafka.common.record; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.utils.CloseableIterator; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.test.TestUtils; import org.junit.Test; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class DefaultRecordBatchTest { @Test public void buildDefaultRecordBatch() { ByteBuffer buffer = ByteBuffer.allocate(2048); MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.CREATE_TIME, 1234567L); builder.appendWithOffset(1234567, 1L, "a".getBytes(), "v".getBytes()); builder.appendWithOffset(1234568, 2L, "b".getBytes(), "v".getBytes()); MemoryRecords records = builder.build(); for (MutableRecordBatch batch : records.batches()) { assertTrue(batch.isValid()); assertEquals(1234567, batch.baseOffset()); assertEquals(1234568, batch.lastOffset()); assertEquals(2L, batch.maxTimestamp()); assertEquals(RecordBatch.NO_PRODUCER_ID, batch.producerId()); assertEquals(RecordBatch.NO_PRODUCER_EPOCH, batch.producerEpoch()); assertEquals(RecordBatch.NO_SEQUENCE, batch.baseSequence()); assertEquals(RecordBatch.NO_SEQUENCE, batch.lastSequence()); for (Record record : batch) { assertTrue(record.isValid()); } } } @Test public void buildDefaultRecordBatchWithProducerId() { long pid = 23423L; short epoch = 145; int baseSequence = 983; ByteBuffer buffer = ByteBuffer.allocate(2048); MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.CREATE_TIME, 1234567L, RecordBatch.NO_TIMESTAMP, pid, epoch, baseSequence); builder.appendWithOffset(1234567, 1L, "a".getBytes(), "v".getBytes()); builder.appendWithOffset(1234568, 2L, "b".getBytes(), "v".getBytes()); MemoryRecords records = builder.build(); for (MutableRecordBatch batch : records.batches()) { assertTrue(batch.isValid()); assertEquals(1234567, batch.baseOffset()); assertEquals(1234568, batch.lastOffset()); assertEquals(2L, batch.maxTimestamp()); assertEquals(pid, batch.producerId()); assertEquals(epoch, batch.producerEpoch()); assertEquals(baseSequence, batch.baseSequence()); assertEquals(baseSequence + 1, batch.lastSequence()); for (Record record : batch) { assertTrue(record.isValid()); } } } @Test public void testSizeInBytes() { Header[] headers = new Header[] { new RecordHeader("foo", "value".getBytes()), new RecordHeader("bar", (byte[]) null) }; long timestamp = System.currentTimeMillis(); SimpleRecord[] records = new SimpleRecord[] { new SimpleRecord(timestamp, "key".getBytes(), "value".getBytes()), new SimpleRecord(timestamp + 30000, null, "value".getBytes()), new SimpleRecord(timestamp + 60000, "key".getBytes(), null), new SimpleRecord(timestamp + 60000, "key".getBytes(), "value".getBytes(), headers) }; int actualSize = MemoryRecords.withRecords(CompressionType.NONE, records).sizeInBytes(); assertEquals(actualSize, DefaultRecordBatch.sizeInBytes(Arrays.asList(records))); } @Test(expected = InvalidRecordException.class) public void testInvalidRecordSize() { MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L, CompressionType.NONE, TimestampType.CREATE_TIME, new SimpleRecord(1L, "a".getBytes(), "1".getBytes()), new SimpleRecord(2L, "b".getBytes(), "2".getBytes()), new SimpleRecord(3L, "c".getBytes(), "3".getBytes())); ByteBuffer buffer = records.buffer(); buffer.putInt(DefaultRecordBatch.LENGTH_OFFSET, 10); DefaultRecordBatch batch = new DefaultRecordBatch(buffer); assertFalse(batch.isValid()); batch.ensureValid(); } @Test(expected = InvalidRecordException.class) public void testInvalidCrc() { MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L, CompressionType.NONE, TimestampType.CREATE_TIME, new SimpleRecord(1L, "a".getBytes(), "1".getBytes()), new SimpleRecord(2L, "b".getBytes(), "2".getBytes()), new SimpleRecord(3L, "c".getBytes(), "3".getBytes())); ByteBuffer buffer = records.buffer(); buffer.putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, 23); DefaultRecordBatch batch = new DefaultRecordBatch(buffer); assertFalse(batch.isValid()); batch.ensureValid(); } @Test public void testSetLastOffset() { SimpleRecord[] simpleRecords = new SimpleRecord[] { new SimpleRecord(1L, "a".getBytes(), "1".getBytes()), new SimpleRecord(2L, "b".getBytes(), "2".getBytes()), new SimpleRecord(3L, "c".getBytes(), "3".getBytes()) }; MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L, CompressionType.NONE, TimestampType.CREATE_TIME, simpleRecords); long lastOffset = 500L; long firstOffset = lastOffset - simpleRecords.length + 1; DefaultRecordBatch batch = new DefaultRecordBatch(records.buffer()); batch.setLastOffset(lastOffset); assertEquals(lastOffset, batch.lastOffset()); assertEquals(firstOffset, batch.baseOffset()); assertTrue(batch.isValid()); List<MutableRecordBatch> recordBatches = Utils.toList(records.batches().iterator()); assertEquals(1, recordBatches.size()); assertEquals(lastOffset, recordBatches.get(0).lastOffset()); long offset = firstOffset; for (Record record : records.records()) assertEquals(offset++, record.offset()); } @Test public void testSetPartitionLeaderEpoch() { MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L, CompressionType.NONE, TimestampType.CREATE_TIME, new SimpleRecord(1L, "a".getBytes(), "1".getBytes()), new SimpleRecord(2L, "b".getBytes(), "2".getBytes()), new SimpleRecord(3L, "c".getBytes(), "3".getBytes())); int leaderEpoch = 500; DefaultRecordBatch batch = new DefaultRecordBatch(records.buffer()); batch.setPartitionLeaderEpoch(leaderEpoch); assertEquals(leaderEpoch, batch.partitionLeaderEpoch()); assertTrue(batch.isValid()); List<MutableRecordBatch> recordBatches = Utils.toList(records.batches().iterator()); assertEquals(1, recordBatches.size()); assertEquals(leaderEpoch, recordBatches.get(0).partitionLeaderEpoch()); } @Test public void testSetLogAppendTime() { MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L, CompressionType.NONE, TimestampType.CREATE_TIME, new SimpleRecord(1L, "a".getBytes(), "1".getBytes()), new SimpleRecord(2L, "b".getBytes(), "2".getBytes()), new SimpleRecord(3L, "c".getBytes(), "3".getBytes())); long logAppendTime = 15L; DefaultRecordBatch batch = new DefaultRecordBatch(records.buffer()); batch.setMaxTimestamp(TimestampType.LOG_APPEND_TIME, logAppendTime); assertEquals(TimestampType.LOG_APPEND_TIME, batch.timestampType()); assertEquals(logAppendTime, batch.maxTimestamp()); assertTrue(batch.isValid()); List<MutableRecordBatch> recordBatches = Utils.toList(records.batches().iterator()); assertEquals(1, recordBatches.size()); assertEquals(logAppendTime, recordBatches.get(0).maxTimestamp()); assertEquals(TimestampType.LOG_APPEND_TIME, recordBatches.get(0).timestampType()); for (Record record : records.records()) assertEquals(logAppendTime, record.timestamp()); } @Test(expected = IllegalArgumentException.class) public void testSetNoTimestampTypeNotAllowed() { MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L, CompressionType.NONE, TimestampType.CREATE_TIME, new SimpleRecord(1L, "a".getBytes(), "1".getBytes()), new SimpleRecord(2L, "b".getBytes(), "2".getBytes()), new SimpleRecord(3L, "c".getBytes(), "3".getBytes())); DefaultRecordBatch batch = new DefaultRecordBatch(records.buffer()); batch.setMaxTimestamp(TimestampType.NO_TIMESTAMP_TYPE, RecordBatch.NO_TIMESTAMP); } @Test public void testReadAndWriteControlBatch() { long producerId = 1L; short producerEpoch = 0; int coordinatorEpoch = 15; ByteBuffer buffer = ByteBuffer.allocate(128); MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME, 0L, RecordBatch.NO_TIMESTAMP, producerId, producerEpoch, RecordBatch.NO_SEQUENCE, true, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.remaining()); EndTransactionMarker marker = new EndTransactionMarker(ControlRecordType.COMMIT, coordinatorEpoch); builder.appendEndTxnMarker(System.currentTimeMillis(), marker); MemoryRecords records = builder.build(); List<MutableRecordBatch> batches = TestUtils.toList(records.batches()); assertEquals(1, batches.size()); MutableRecordBatch batch = batches.get(0); assertTrue(batch.isControlBatch()); List<Record> logRecords = TestUtils.toList(records.records()); assertEquals(1, logRecords.size()); Record commitRecord = logRecords.get(0); assertEquals(marker, EndTransactionMarker.deserialize(commitRecord)); } @Test public void testStreamingIteratorConsistency() { MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L, CompressionType.GZIP, TimestampType.CREATE_TIME, new SimpleRecord(1L, "a".getBytes(), "1".getBytes()), new SimpleRecord(2L, "b".getBytes(), "2".getBytes()), new SimpleRecord(3L, "c".getBytes(), "3".getBytes())); DefaultRecordBatch batch = new DefaultRecordBatch(records.buffer()); try (CloseableIterator<Record> streamingIterator = batch.streamingIterator()) { TestUtils.checkEquals(streamingIterator, batch.iterator()); } } }