/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.morphlines.solr; import java.io.File; import java.io.IOException; import java.util.Iterator; import java.util.Locale; import com.codahale.metrics.MetricRegistry; import com.google.common.collect.ListMultimap; import com.typesafe.config.Config; import org.apache.lucene.util.Constants; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.AbstractDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocument; import org.junit.Before; import org.junit.BeforeClass; import org.kitesdk.morphline.api.Collector; import org.kitesdk.morphline.api.Command; import org.kitesdk.morphline.api.MorphlineContext; import org.kitesdk.morphline.api.Record; import org.kitesdk.morphline.base.Compiler; import org.kitesdk.morphline.base.FaultTolerance; import org.kitesdk.morphline.base.Notifications; import org.kitesdk.morphline.stdlib.PipeBuilder; public abstract class AbstractSolrMorphlineZkTestBase extends SolrCloudTestCase { protected static final String COLLECTION = "collection1"; protected static final int TIMEOUT = 30; @BeforeClass public static void setupCluster() throws Exception { configureCluster(2) .addConfig("conf", SOLR_CONF_DIR.toPath()) .configure(); CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1) .processAndWait(cluster.getSolrClient(), TIMEOUT); AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(), false, true, TIMEOUT); } protected static final String RESOURCES_DIR = getFile("morphlines-core.marker").getParent(); private static final File SOLR_CONF_DIR = new File(RESOURCES_DIR + "/solr/collection1/conf"); protected Collector collector; protected Command morphline; @BeforeClass public static void setupClass() throws Exception { assumeFalse("This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8876)", Constants.JRE_IS_MINIMUM_JAVA9); assumeFalse("This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)", new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage())); } @Before public void setup() throws Exception { collector = new Collector(); } protected void commit() throws Exception { Notifications.notifyCommitTransaction(morphline); } protected Command parse(String file) throws IOException { return parse(file, COLLECTION); } protected Command parse(String file, String collection) throws IOException { SolrLocator locator = new SolrLocator(createMorphlineContext()); locator.setCollectionName(collection); locator.setZkHost(cluster.getZkServer().getZkAddress()); //locator.setServerUrl(cloudJettys.get(0).url); // TODO: download IndexSchema from solrUrl not yet implemented //locator.setSolrHomeDir(SOLR_HOME_DIR.getPath()); Config config = new Compiler().parse(new File(RESOURCES_DIR + "/" + file + ".conf"), locator.toConfig("SOLR_LOCATOR")); config = config.getConfigList("morphlines").get(0); return createMorphline(config); } private Command createMorphline(Config config) { return new PipeBuilder().build(config, null, collector, createMorphlineContext()); } private MorphlineContext createMorphlineContext() { return new MorphlineContext.Builder() .setExceptionHandler(new FaultTolerance(false, false, SolrServerException.class.getName())) .setMetricRegistry(new MetricRegistry()) .build(); } protected void startSession() { Notifications.notifyStartSession(morphline); } protected ListMultimap<String, Object> next(Iterator<SolrDocument> iter) { SolrDocument doc = iter.next(); Record record = toRecord(doc); record.removeAll("_version_"); // the values of this field are unknown and internal to solr return record.getFields(); } private Record toRecord(SolrDocument doc) { Record record = new Record(); for (String key : doc.keySet()) { record.getFields().replaceValues(key, doc.getFieldValues(key)); } return record; } }