/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * <p/> * http://www.apache.org/licenses/LICENSE-2.0 * <p/> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.cassandra; import java.io.File; import java.io.IOException; import java.util.*; import org.junit.After; import org.junit.BeforeClass; import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.statements.IndexTarget; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.index.StubIndex; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.schema.*; import org.apache.cassandra.service.MigrationManager; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; public class SchemaLoader { @BeforeClass public static void loadSchema() throws ConfigurationException { prepareServer(); // Migrations aren't happy if gossiper is not started. Even if we don't use migrations though, // some tests now expect us to start gossip for them. startGossiper(); } @After public void leakDetect() throws InterruptedException { System.gc(); System.gc(); System.gc(); Thread.sleep(10); } public static void prepareServer() { CQLTester.prepareServer(); } public static void startGossiper() { if (!Gossiper.instance.isEnabled()) Gossiper.instance.start((int) (System.currentTimeMillis() / 1000)); } public static void schemaDefinition(String testName) throws ConfigurationException { List<KeyspaceMetadata> schema = new ArrayList<KeyspaceMetadata>(); // A whole bucket of shorthand String ks1 = testName + "Keyspace1"; String ks2 = testName + "Keyspace2"; String ks3 = testName + "Keyspace3"; String ks4 = testName + "Keyspace4"; String ks5 = testName + "Keyspace5"; String ks6 = testName + "Keyspace6"; String ks_kcs = testName + "KeyCacheSpace"; String ks_rcs = testName + "RowCacheSpace"; String ks_ccs = testName + "CounterCacheSpace"; String ks_nocommit = testName + "NoCommitlogSpace"; String ks_prsi = testName + "PerRowSecondaryIndex"; String ks_cql = testName + "cql_keyspace"; AbstractType bytes = BytesType.instance; AbstractType<?> composite = CompositeType.getInstance(Arrays.asList(new AbstractType<?>[]{BytesType.instance, TimeUUIDType.instance, IntegerType.instance})); AbstractType<?> compositeMaxMin = CompositeType.getInstance(Arrays.asList(new AbstractType<?>[]{BytesType.instance, IntegerType.instance})); Map<Byte, AbstractType<?>> aliases = new HashMap<Byte, AbstractType<?>>(); aliases.put((byte)'b', BytesType.instance); aliases.put((byte)'t', TimeUUIDType.instance); aliases.put((byte)'B', ReversedType.getInstance(BytesType.instance)); aliases.put((byte)'T', ReversedType.getInstance(TimeUUIDType.instance)); AbstractType<?> dynamicComposite = DynamicCompositeType.getInstance(aliases); // Make it easy to test compaction Map<String, String> compactionOptions = new HashMap<String, String>(); compactionOptions.put("tombstone_compaction_interval", "1"); Map<String, String> leveledOptions = new HashMap<String, String>(); leveledOptions.put("sstable_size_in_mb", "1"); // Keyspace 1 schema.add(KeyspaceMetadata.create(ks1, KeyspaceParams.simple(1), Tables.of( // Column Families standardCFMD(ks1, "Standard1").compaction(CompactionParams.scts(compactionOptions)), standardCFMD(ks1, "Standard2"), standardCFMD(ks1, "Standard3"), standardCFMD(ks1, "Standard4"), standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0), standardCFMD(ks1, "StandardLong1"), standardCFMD(ks1, "StandardLong2"), //CFMetaData.Builder.create(ks1, "ValuesWithQuotes").build(), superCFMD(ks1, "Super1", LongType.instance), superCFMD(ks1, "Super2", LongType.instance), superCFMD(ks1, "Super3", LongType.instance), superCFMD(ks1, "Super4", UTF8Type.instance), superCFMD(ks1, "Super5", bytes), superCFMD(ks1, "Super6", LexicalUUIDType.instance, UTF8Type.instance), keysIndexCFMD(ks1, "Indexed1", true), keysIndexCFMD(ks1, "Indexed2", false), //CFMetaData.Builder.create(ks1, "StandardInteger1").withColumnNameComparator(IntegerType.instance).build(), //CFMetaData.Builder.create(ks1, "StandardLong3").withColumnNameComparator(IntegerType.instance).build(), //CFMetaData.Builder.create(ks1, "Counter1", false, false, true).build(), //CFMetaData.Builder.create(ks1, "SuperCounter1", false, false, true, true).build(), superCFMD(ks1, "SuperDirectGC", BytesType.instance).gcGraceSeconds(0), // jdbcCFMD(ks1, "JdbcInteger", IntegerType.instance).addColumnDefinition(integerColumn(ks1, "JdbcInteger")), jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance).addColumnDefinition(utf8Column(ks1, "JdbcUtf8")), jdbcCFMD(ks1, "JdbcLong", LongType.instance), jdbcCFMD(ks1, "JdbcBytes", bytes), jdbcCFMD(ks1, "JdbcAscii", AsciiType.instance), //CFMetaData.Builder.create(ks1, "StandardComposite", false, true, false).withColumnNameComparator(composite).build(), //CFMetaData.Builder.create(ks1, "StandardComposite2", false, true, false).withColumnNameComparator(compositeMaxMin).build(), //CFMetaData.Builder.create(ks1, "StandardDynamicComposite", false, true, false).withColumnNameComparator(dynamicComposite).build(), standardCFMD(ks1, "StandardLeveled").compaction(CompactionParams.lcs(leveledOptions)), standardCFMD(ks1, "legacyleveled").compaction(CompactionParams.lcs(leveledOptions)), standardCFMD(ks1, "StandardLowIndexInterval").minIndexInterval(8) .maxIndexInterval(256) .caching(CachingParams.CACHE_NOTHING) //CFMetaData.Builder.create(ks1, "UUIDKeys").addPartitionKey("key",UUIDType.instance).build(), //CFMetaData.Builder.create(ks1, "MixedTypes").withColumnNameComparator(LongType.instance).addPartitionKey("key", UUIDType.instance).build(), //CFMetaData.Builder.create(ks1, "MixedTypesComposite", false, true, false).withColumnNameComparator(composite).addPartitionKey("key", composite).build(), //CFMetaData.Builder.create(ks1, "AsciiKeys").addPartitionKey("key", AsciiType.instance).build() ))); // Keyspace 2 schema.add(KeyspaceMetadata.create(ks2, KeyspaceParams.simple(1), Tables.of( // Column Families standardCFMD(ks2, "Standard1"), standardCFMD(ks2, "Standard3"), superCFMD(ks2, "Super3", bytes), superCFMD(ks2, "Super4", TimeUUIDType.instance), keysIndexCFMD(ks2, "Indexed1", true), compositeIndexCFMD(ks2, "Indexed2", true), compositeIndexCFMD(ks2, "Indexed3", true).gcGraceSeconds(0)))); // Keyspace 3 schema.add(KeyspaceMetadata.create(ks3, KeyspaceParams.simple(5), Tables.of( standardCFMD(ks3, "Standard1"), keysIndexCFMD(ks3, "Indexed1", true)))); // Keyspace 4 schema.add(KeyspaceMetadata.create(ks4, KeyspaceParams.simple(3), Tables.of( standardCFMD(ks4, "Standard1"), standardCFMD(ks4, "Standard3"), superCFMD(ks4, "Super3", bytes), superCFMD(ks4, "Super4", TimeUUIDType.instance), superCFMD(ks4, "Super5", TimeUUIDType.instance, BytesType.instance)))); // Keyspace 5 schema.add(KeyspaceMetadata.create(ks5, KeyspaceParams.simple(2), Tables.of(standardCFMD(ks5, "Standard1")))); // Keyspace 6 schema.add(KeyspaceMetadata.create(ks6, KeyspaceParams.simple(1), Tables.of(keysIndexCFMD(ks6, "Indexed1", true)))); // KeyCacheSpace schema.add(KeyspaceMetadata.create(ks_kcs, KeyspaceParams.simple(1), Tables.of( standardCFMD(ks_kcs, "Standard1"), standardCFMD(ks_kcs, "Standard2"), standardCFMD(ks_kcs, "Standard3")))); // RowCacheSpace schema.add(KeyspaceMetadata.create(ks_rcs, KeyspaceParams.simple(1), Tables.of( standardCFMD(ks_rcs, "CFWithoutCache").caching(CachingParams.CACHE_NOTHING), standardCFMD(ks_rcs, "CachedCF").caching(CachingParams.CACHE_EVERYTHING), standardCFMD(ks_rcs, "CachedIntCF"). caching(new CachingParams(true, 100))))); // CounterCacheSpace /*schema.add(KeyspaceMetadata.testMetadata(ks_ccs, simple, opts_rf1, CFMetaData.Builder.create(ks_ccs, "Counter1", false, false, true).build(), CFMetaData.Builder.create(ks_ccs, "Counter1", false, false, true).build()));*/ schema.add(KeyspaceMetadata.create(ks_nocommit, KeyspaceParams.simpleTransient(1), Tables.of( standardCFMD(ks_nocommit, "Standard1")))); // CQLKeyspace schema.add(KeyspaceMetadata.create(ks_cql, KeyspaceParams.simple(1), Tables.of( // Column Families CFMetaData.compile("CREATE TABLE table1 (" + "k int PRIMARY KEY," + "v1 text," + "v2 int" + ")", ks_cql), CFMetaData.compile("CREATE TABLE table2 (" + "k text," + "c text," + "v text," + "PRIMARY KEY (k, c))", ks_cql), CFMetaData.compile("CREATE TABLE foo (" + "bar text, " + "baz text, " + "qux text, " + "PRIMARY KEY(bar, baz) ) " + "WITH COMPACT STORAGE", ks_cql), CFMetaData.compile("CREATE TABLE foofoo (" + "bar text, " + "baz text, " + "qux text, " + "quz text, " + "foo text, " + "PRIMARY KEY((bar, baz), qux, quz) ) " + "WITH COMPACT STORAGE", ks_cql) ))); if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false"))) useCompression(schema); // if you're messing with low-level sstable stuff, it can be useful to inject the schema directly // Schema.instance.load(schemaDefinition()); for (KeyspaceMetadata ksm : schema) MigrationManager.announceNewKeyspace(ksm, false); } public static void createKeyspace(String name, KeyspaceParams params, CFMetaData... tables) { MigrationManager.announceNewKeyspace(KeyspaceMetadata.create(name, params, Tables.of(tables)), true); } public static ColumnDefinition integerColumn(String ksName, String cfName) { return new ColumnDefinition(ksName, cfName, ColumnIdentifier.getInterned(IntegerType.instance.fromString("42"), IntegerType.instance), UTF8Type.instance, ColumnDefinition.NO_POSITION, ColumnDefinition.Kind.REGULAR); } public static ColumnDefinition utf8Column(String ksName, String cfName) { return new ColumnDefinition(ksName, cfName, ColumnIdentifier.getInterned("fortytwo", true), UTF8Type.instance, ColumnDefinition.NO_POSITION, ColumnDefinition.Kind.REGULAR); } public static CFMetaData perRowIndexedCFMD(String ksName, String cfName) { final Map<String, String> indexOptions = Collections.singletonMap( IndexTarget.CUSTOM_INDEX_OPTION_NAME, StubIndex.class.getName()); CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName) .addPartitionKey("key", AsciiType.instance) .build(); ColumnDefinition indexedColumn = ColumnDefinition.regularDef(ksName, cfName, "indexed", AsciiType.instance); cfm.addOrReplaceColumnDefinition(indexedColumn); cfm.indexes( cfm.getIndexes() .with(IndexMetadata.fromIndexTargets(cfm, Collections.singletonList(new IndexTarget(indexedColumn.name, IndexTarget.Type.VALUES)), "indexe1", IndexMetadata.Kind.CUSTOM, indexOptions))); return cfm; } private static void useCompression(List<KeyspaceMetadata> schema) { for (KeyspaceMetadata ksm : schema) for (CFMetaData cfm : ksm.tablesAndViews()) cfm.compression(CompressionParams.snappy()); } public static CFMetaData counterCFMD(String ksName, String cfName) { return CFMetaData.Builder.create(ksName, cfName, false, true, true) .addPartitionKey("key", AsciiType.instance) .addClusteringColumn("name", AsciiType.instance) .addRegularColumn("val", CounterColumnType.instance) .addRegularColumn("val2", CounterColumnType.instance) .build() .compression(getCompressionParameters()); } public static CFMetaData standardCFMD(String ksName, String cfName) { return standardCFMD(ksName, cfName, 1, AsciiType.instance); } public static CFMetaData standardCFMD(String ksName, String cfName, int columnCount, AbstractType<?> keyType) { return standardCFMD(ksName, cfName, columnCount, keyType, AsciiType.instance); } public static CFMetaData standardCFMD(String ksName, String cfName, int columnCount, AbstractType<?> keyType, AbstractType<?> valType) { return standardCFMD(ksName, cfName, columnCount, keyType, valType, AsciiType.instance); } public static CFMetaData standardCFMD(String ksName, String cfName, int columnCount, AbstractType<?> keyType, AbstractType<?> valType, AbstractType<?> clusteringType) { CFMetaData.Builder builder = CFMetaData.Builder.create(ksName, cfName) .addPartitionKey("key", keyType) .addClusteringColumn("name", clusteringType) .addRegularColumn("val", valType); for (int i = 0; i < columnCount; i++) builder.addRegularColumn("val" + i, AsciiType.instance); return builder.build() .compression(getCompressionParameters()); } public static CFMetaData denseCFMD(String ksName, String cfName) { return denseCFMD(ksName, cfName, AsciiType.instance); } public static CFMetaData denseCFMD(String ksName, String cfName, AbstractType cc) { return denseCFMD(ksName, cfName, cc, null); } public static CFMetaData denseCFMD(String ksName, String cfName, AbstractType cc, AbstractType subcc) { AbstractType comp = cc; if (subcc != null) comp = CompositeType.getInstance(Arrays.asList(new AbstractType<?>[]{cc, subcc})); return CFMetaData.Builder.createDense(ksName, cfName, subcc != null, false) .addPartitionKey("key", AsciiType.instance) .addClusteringColumn("cols", comp) .addRegularColumn("val", AsciiType.instance) .build() .compression(getCompressionParameters()); } // TODO: Fix superCFMD failing on legacy table creation. Seems to be applying composite comparator to partition key public static CFMetaData superCFMD(String ksName, String cfName, AbstractType subcc) { return superCFMD(ksName, cfName, BytesType.instance, subcc); } public static CFMetaData superCFMD(String ksName, String cfName, AbstractType cc, AbstractType subcc) { return superCFMD(ksName, cfName, "cols", cc, subcc); } public static CFMetaData superCFMD(String ksName, String cfName, String ccName, AbstractType cc, AbstractType subcc) { //This is busted // return CFMetaData.Builder.createSuper(ksName, cfName, false) // .addPartitionKey("0", BytesType.instance) // .addClusteringColumn("1", cc) // .addClusteringColumn("2", subcc) // .addRegularColumn("3", AsciiType.instance) // .build(); return standardCFMD(ksName, cfName); } public static CFMetaData compositeIndexCFMD(String ksName, String cfName, boolean withIndex) throws ConfigurationException { // the withIndex flag exists to allow tests index creation // on existing columns CFMetaData cfm = CFMetaData.Builder.create(ksName, cfName) .addPartitionKey("key", AsciiType.instance) .addClusteringColumn("c1", AsciiType.instance) .addRegularColumn("birthdate", LongType.instance) .addRegularColumn("notbirthdate", LongType.instance) .build(); if (withIndex) cfm.indexes( cfm.getIndexes() .with(IndexMetadata.fromIndexTargets(cfm, Collections.singletonList( new IndexTarget(new ColumnIdentifier("birthdate", true), IndexTarget.Type.VALUES)), "birthdate_key_index", IndexMetadata.Kind.COMPOSITES, Collections.EMPTY_MAP))); return cfm.compression(getCompressionParameters()); } public static CFMetaData keysIndexCFMD(String ksName, String cfName, boolean withIndex) throws ConfigurationException { CFMetaData cfm = CFMetaData.Builder.createDense(ksName, cfName, false, false) .addPartitionKey("key", AsciiType.instance) .addClusteringColumn("c1", AsciiType.instance) .addStaticColumn("birthdate", LongType.instance) .addStaticColumn("notbirthdate", LongType.instance) .addRegularColumn("value", LongType.instance) .build(); if (withIndex) cfm.indexes( cfm.getIndexes() .with(IndexMetadata.fromIndexTargets(cfm, Collections.singletonList( new IndexTarget(new ColumnIdentifier("birthdate", true), IndexTarget.Type.VALUES)), "birthdate_composite_index", IndexMetadata.Kind.KEYS, Collections.EMPTY_MAP))); return cfm.compression(getCompressionParameters()); } public static CFMetaData jdbcCFMD(String ksName, String cfName, AbstractType comp) { return CFMetaData.Builder.create(ksName, cfName).addPartitionKey("key", BytesType.instance) .build() .compression(getCompressionParameters()); } public static CompressionParams getCompressionParameters() { return getCompressionParameters(null); } public static CompressionParams getCompressionParameters(Integer chunkSize) { if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false"))) return CompressionParams.snappy(chunkSize); return CompressionParams.noCompression(); } public static void cleanupAndLeaveDirs() throws IOException { // We need to stop and unmap all CLS instances prior to cleanup() or we'll get failures on Windows. CommitLog.instance.stopUnsafe(true); mkdirs(); cleanup(); mkdirs(); CommitLog.instance.restartUnsafe(); } public static void cleanup() { // clean up commitlog String[] directoryNames = { DatabaseDescriptor.getCommitLogLocation(), }; for (String dirName : directoryNames) { File dir = new File(dirName); if (!dir.exists()) throw new RuntimeException("No such directory: " + dir.getAbsolutePath()); // Leave the folder around as Windows will complain about directory deletion w/handles open to children files String[] children = dir.list(); for (String child : children) FileUtils.deleteRecursive(new File(dir, child)); } cleanupSavedCaches(); // clean up data directory which are stored as data directory/keyspace/data files for (String dirName : DatabaseDescriptor.getAllDataFileLocations()) { File dir = new File(dirName); if (!dir.exists()) throw new RuntimeException("No such directory: " + dir.getAbsolutePath()); String[] children = dir.list(); for (String child : children) FileUtils.deleteRecursive(new File(dir, child)); } } public static void mkdirs() { DatabaseDescriptor.createAllDirectories(); } public static void insertData(String keyspace, String columnFamily, int offset, int numberOfRows) { CFMetaData cfm = Schema.instance.getCFMetaData(keyspace, columnFamily); for (int i = offset; i < offset + numberOfRows; i++) { RowUpdateBuilder builder = new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), ByteBufferUtil.bytes("key"+i)); builder.clustering(ByteBufferUtil.bytes("col"+ i)).add("val", ByteBufferUtil.bytes("val" + i)); builder.build().apply(); } } public static void cleanupSavedCaches() { File cachesDir = new File(DatabaseDescriptor.getSavedCachesLocation()); if (!cachesDir.exists() || !cachesDir.isDirectory()) return; FileUtils.delete(cachesDir.listFiles()); } }