/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.hadoop;
import java.net.URLEncoder;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.types.Pair;
import org.apache.lucene.util.Constants;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.hadoop.morphline.MorphlineMapper;
import org.apache.solr.util.BadHdfsThreadsFilter;
import org.junit.BeforeClass;
import org.junit.Test;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
public class MorphlineMapperTest extends MRUnitBase {
@BeforeClass
public static void beforeClass() {
assumeFalse("Does not work on Windows, because it uses UNIX shell commands or POSIX paths", Constants.WINDOWS);
}
@Test
@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-9220")
public void testMapper() throws Exception {
MorphlineMapper mapper = new MorphlineMapper();
MapDriver<LongWritable, Text, Text, SolrInputDocumentWritable> mapDriver = MapDriver.newMapDriver(mapper);;
Configuration config = mapDriver.getConfiguration();
setupHadoopConfig(config);
mapDriver.withInput(new LongWritable(0L), new Text("hdfs://localhost/" +
URLEncoder.encode(DOCUMENTS_DIR, "UTF-8").replace("+", "%20") +
"/sample-statuses-20120906-141433.avro"));
SolrInputDocument sid = new SolrInputDocument();
sid.addField("id", "uniqueid1");
sid.addField("user_name", "user1");
sid.addField("text", "content of record one");
SolrInputDocumentWritable sidw = new SolrInputDocumentWritable(sid);
mapDriver
.withCacheArchive(solrHomeZip.getAbsolutePath())
.withOutput(new Text("0"), sidw);
//mapDriver.runTest();
List<Pair<Text, SolrInputDocumentWritable>> result = mapDriver.run();
for (Pair<Text, SolrInputDocumentWritable> p: result) {
System.out.println(p.getFirst());
System.out.println(p.getSecond());
}
}
}