/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cassandra.service;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.util.*;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.cassandra.*;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.partitions.FilteredPartition;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.pager.QueryPager;
import org.apache.cassandra.service.pager.PagingState;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.transport.Server;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
import static org.junit.Assert.*;
@RunWith(OrderedJUnit4ClassRunner.class)
public class QueryPagerTest
{
public static final String KEYSPACE1 = "QueryPagerTest";
public static final String CF_STANDARD = "Standard1";
public static final String KEYSPACE_CQL = "cql_keyspace";
public static final String CF_CQL = "table2";
public static final int nowInSec = FBUtilities.nowInSeconds();
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD));
SchemaLoader.createKeyspace(KEYSPACE_CQL,
KeyspaceParams.simple(1),
CFMetaData.compile("CREATE TABLE " + CF_CQL + " ("
+ "k text,"
+ "c text,"
+ "v text,"
+ "PRIMARY KEY (k, c))", KEYSPACE_CQL));
addData();
}
private static String string(ByteBuffer bb)
{
try
{
return ByteBufferUtil.string(bb);
}
catch (CharacterCodingException e)
{
throw new RuntimeException(e);
}
}
public static void addData()
{
cfs().clearUnsafe();
int nbKeys = 10;
int nbCols = 10;
// *
// * Creates the following data:
// * k1: c1 ... cn
// * ...
// * ki: c1 ... cn
// *
for (int i = 0; i < nbKeys; i++)
{
for (int j = 0; j < nbCols; j++)
{
RowUpdateBuilder builder = new RowUpdateBuilder(cfs().metadata, FBUtilities.timestampMicros(), "k" + i);
builder.clustering("c" + j).add("val", "").build().applyUnsafe();
}
}
}
private static ColumnFamilyStore cfs()
{
return Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
}
private static List<FilteredPartition> query(QueryPager pager, int expectedSize)
{
return query(pager, expectedSize, expectedSize);
}
private static List<FilteredPartition> query(QueryPager pager, int toQuery, int expectedSize)
{
StringBuilder sb = new StringBuilder();
List<FilteredPartition> partitionList = new ArrayList<>();
int rows = 0;
try (ReadOrderGroup orderGroup = pager.startOrderGroup(); PartitionIterator iterator = pager.fetchPageInternal(toQuery, orderGroup))
{
while (iterator.hasNext())
{
try (RowIterator rowIter = iterator.next())
{
FilteredPartition partition = FilteredPartition.create(rowIter);
sb.append(partition);
partitionList.add(partition);
rows += partition.rowCount();
}
}
}
assertEquals(sb.toString(), expectedSize, rows);
return partitionList;
}
private static ReadCommand namesQuery(String key, String... names)
{
AbstractReadCommandBuilder builder = Util.cmd(cfs(), key);
for (String name : names)
builder.includeRow(name);
return builder.withPagingLimit(100).build();
}
private static SinglePartitionReadCommand sliceQuery(String key, String start, String end, int count)
{
return sliceQuery(key, start, end, false, count);
}
private static SinglePartitionReadCommand sliceQuery(String key, String start, String end, boolean reversed, int count)
{
ClusteringComparator cmp = cfs().getComparator();
CFMetaData metadata = cfs().metadata;
Slice slice = Slice.make(cmp.make(start), cmp.make(end));
ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(Slices.with(cmp, slice), reversed);
return SinglePartitionReadCommand.create(cfs().metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, Util.dk(key), filter);
}
private static ReadCommand rangeNamesQuery(String keyStart, String keyEnd, int count, String... names)
{
AbstractReadCommandBuilder builder = Util.cmd(cfs())
.fromKeyExcl(keyStart)
.toKeyIncl(keyEnd)
.withPagingLimit(count);
for (String name : names)
builder.includeRow(name);
return builder.build();
}
private static ReadCommand rangeSliceQuery(String keyStart, String keyEnd, int count, String start, String end)
{
return Util.cmd(cfs())
.fromKeyExcl(keyStart)
.toKeyIncl(keyEnd)
.fromIncl(start)
.toIncl(end)
.withPagingLimit(count)
.build();
}
private static void assertRow(FilteredPartition r, String key, String... names)
{
ByteBuffer[] bbs = new ByteBuffer[names.length];
for (int i = 0; i < names.length; i++)
bbs[i] = bytes(names[i]);
assertRow(r, key, bbs);
}
private static void assertRow(FilteredPartition partition, String key, ByteBuffer... names)
{
assertEquals(key, string(partition.partitionKey().getKey()));
assertFalse(partition.isEmpty());
int i = 0;
for (Row row : Util.once(partition.iterator()))
{
ByteBuffer expected = names[i++];
assertEquals("column " + i + " doesn't match "+string(expected)+" vs "+string(row.clustering().get(0)), expected, row.clustering().get(0));
}
}
private QueryPager maybeRecreate(QueryPager pager, ReadQuery command, boolean testPagingState, int protocolVersion)
{
if (!testPagingState)
return pager;
PagingState state = PagingState.deserialize(pager.state().serialize(protocolVersion), protocolVersion);
return command.getPager(state, protocolVersion);
}
@Test
public void namesQueryTest() throws Exception
{
QueryPager pager = namesQuery("k0", "c1", "c5", "c7", "c8").getPager(null, Server.CURRENT_VERSION);
assertFalse(pager.isExhausted());
List<FilteredPartition> partition = query(pager, 5, 4);
assertRow(partition.get(0), "k0", "c1", "c5", "c7", "c8");
assertTrue(pager.isExhausted());
}
@Test
public void sliceQueryTest() throws Exception
{
sliceQueryTest(false, Server.VERSION_3);
sliceQueryTest(true, Server.VERSION_4);
sliceQueryTest(false, Server.VERSION_3);
sliceQueryTest(true, Server.VERSION_4);
}
public void sliceQueryTest(boolean testPagingState, int protocolVersion) throws Exception
{
ReadCommand command = sliceQuery("k0", "c1", "c8", 10);
QueryPager pager = command.getPager(null, protocolVersion);
assertFalse(pager.isExhausted());
List<FilteredPartition> partition = query(pager, 3);
assertRow(partition.get(0), "k0", "c1", "c2", "c3");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partition = query(pager, 3);
assertRow(partition.get(0), "k0", "c4", "c5", "c6");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partition = query(pager, 3, 2);
assertRow(partition.get(0), "k0", "c7", "c8");
assertTrue(pager.isExhausted());
}
@Test
public void reversedSliceQueryTest() throws Exception
{
reversedSliceQueryTest(false, Server.VERSION_3);
reversedSliceQueryTest(true, Server.VERSION_4);
reversedSliceQueryTest(false, Server.VERSION_3);
reversedSliceQueryTest(true, Server.VERSION_4);
}
public void reversedSliceQueryTest(boolean testPagingState, int protocolVersion) throws Exception
{
ReadCommand command = sliceQuery("k0", "c1", "c8", true, 10);
QueryPager pager = command.getPager(null, protocolVersion);
assertFalse(pager.isExhausted());
List<FilteredPartition> partition = query(pager, 3);
assertRow(partition.get(0), "k0", "c6", "c7", "c8");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partition = query(pager, 3);
assertRow(partition.get(0), "k0", "c3", "c4", "c5");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partition = query(pager, 3, 2);
assertRow(partition.get(0), "k0", "c1", "c2");
assertTrue(pager.isExhausted());
}
@Test
public void multiQueryTest() throws Exception
{
multiQueryTest(false, Server.VERSION_3);
multiQueryTest(true, Server.VERSION_4);
multiQueryTest(false, Server.VERSION_3);
multiQueryTest(true, Server.VERSION_4);
}
public void multiQueryTest(boolean testPagingState, int protocolVersion) throws Exception
{
ReadQuery command = new SinglePartitionReadCommand.Group(new ArrayList<SinglePartitionReadCommand>()
{{
add(sliceQuery("k1", "c2", "c6", 10));
add(sliceQuery("k4", "c3", "c5", 10));
}}, DataLimits.NONE);
QueryPager pager = command.getPager(null, protocolVersion);
assertFalse(pager.isExhausted());
List<FilteredPartition> partition = query(pager, 3);
assertRow(partition.get(0), "k1", "c2", "c3", "c4");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partition = query(pager , 4);
assertRow(partition.get(0), "k1", "c5", "c6");
assertRow(partition.get(1), "k4", "c3", "c4");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partition = query(pager, 3, 1);
assertRow(partition.get(0), "k4", "c5");
assertTrue(pager.isExhausted());
}
@Test
public void rangeNamesQueryTest() throws Exception
{
rangeNamesQueryTest(false, Server.VERSION_3);
rangeNamesQueryTest(true, Server.VERSION_4);
rangeNamesQueryTest(false, Server.VERSION_3);
rangeNamesQueryTest(true, Server.VERSION_4);
}
public void rangeNamesQueryTest(boolean testPagingState, int protocolVersion) throws Exception
{
ReadCommand command = rangeNamesQuery("k0", "k5", 100, "c1", "c4", "c8");
QueryPager pager = command.getPager(null, protocolVersion);
assertFalse(pager.isExhausted());
List<FilteredPartition> partitions = query(pager, 3 * 3);
for (int i = 1; i <= 3; i++)
assertRow(partitions.get(i-1), "k" + i, "c1", "c4", "c8");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partitions = query(pager, 3 * 3, 2 * 3);
for (int i = 4; i <= 5; i++)
assertRow(partitions.get(i-4), "k" + i, "c1", "c4", "c8");
assertTrue(pager.isExhausted());
}
@Test
public void rangeSliceQueryTest() throws Exception
{
rangeSliceQueryTest(false, Server.VERSION_3);
rangeSliceQueryTest(true, Server.VERSION_4);
rangeSliceQueryTest(false, Server.VERSION_3);
rangeSliceQueryTest(true, Server.VERSION_4);
}
public void rangeSliceQueryTest(boolean testPagingState, int protocolVersion) throws Exception
{
ReadCommand command = rangeSliceQuery("k1", "k5", 100, "c1", "c7");
QueryPager pager = command.getPager(null, protocolVersion);
assertFalse(pager.isExhausted());
List<FilteredPartition> partitions = query(pager, 5);
assertRow(partitions.get(0), "k2", "c1", "c2", "c3", "c4", "c5");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partitions = query(pager, 4);
assertRow(partitions.get(0), "k2", "c6", "c7");
assertRow(partitions.get(1), "k3", "c1", "c2");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partitions = query(pager, 6);
assertRow(partitions.get(0), "k3", "c3", "c4", "c5", "c6", "c7");
assertRow(partitions.get(1), "k4", "c1");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partitions = query(pager, 5);
assertRow(partitions.get(0), "k4", "c2", "c3", "c4", "c5", "c6");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partitions = query(pager, 5);
assertRow(partitions.get(0), "k4", "c7");
assertRow(partitions.get(1), "k5", "c1", "c2", "c3", "c4");
assertFalse(pager.isExhausted());
pager = maybeRecreate(pager, command, testPagingState, protocolVersion);
assertFalse(pager.isExhausted());
partitions = query(pager, 5, 3);
assertRow(partitions.get(0), "k5", "c5", "c6", "c7");
assertTrue(pager.isExhausted());
}
@Test
public void SliceQueryWithTombstoneTest() throws Exception
{
// Testing for the bug of #6748
String keyspace = "cql_keyspace";
String table = "table2";
ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
// Insert rows but with a tombstone as last cell
for (int i = 0; i < 5; i++)
executeInternal(String.format("INSERT INTO %s.%s (k, c, v) VALUES ('k%d', 'c%d', null)", keyspace, table, 0, i));
ReadCommand command = SinglePartitionReadCommand.create(cfs.metadata, nowInSec, Util.dk("k0"), Slice.ALL);
QueryPager pager = command.getPager(null, Server.CURRENT_VERSION);
for (int i = 0; i < 5; i++)
{
List<FilteredPartition> partitions = query(pager, 1);
// The only live cell we should have each time is the row marker
assertRow(partitions.get(0), "k0", "c" + i);
}
}
}