/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.hadoop;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.Locale;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.lucene.util.Constants;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.cloud.AbstractZkTestCase;
import org.apache.solr.hadoop.dedup.NoChangeUpdateConflictResolver;
import org.apache.solr.hadoop.dedup.RetainMostRecentUpdateConflictResolver;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
public class MapReduceIndexerToolArgumentParserTest extends SolrTestCaseJ4 {
private Configuration conf;
private MapReduceIndexerTool.MyArgumentParser parser;
private MapReduceIndexerTool.Options opts;
private PrintStream oldSystemOut;
private PrintStream oldSystemErr;
private ByteArrayOutputStream bout;
private ByteArrayOutputStream berr;
private static final String RESOURCES_DIR = getFile("morphlines-core.marker").getParent();
private static final File MINIMR_INSTANCE_DIR = new File(RESOURCES_DIR + "/solr/minimr");
private static final String MORPHLINE_FILE = RESOURCES_DIR + "/test-morphlines/solrCellDocumentTypes.conf";
private final File solrHomeDirectory = createTempDir().toFile();
@BeforeClass
public static void beforeClass() {
assumeFalse("This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8876)", Constants.JRE_IS_MINIMUM_JAVA9);
assumeFalse("Does not work on Windows, because it uses UNIX shell commands or POSIX paths", Constants.WINDOWS);
assumeFalse("This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)",
new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage()));
}
@Before
public void setUp() throws Exception {
super.setUp();
AbstractZkTestCase.SOLRHOME = solrHomeDirectory;
FileUtils.copyDirectory(MINIMR_INSTANCE_DIR, solrHomeDirectory);
conf = new Configuration();
parser = new MapReduceIndexerTool.MyArgumentParser();
opts = new MapReduceIndexerTool.Options();
oldSystemOut = System.out;
bout = new ByteArrayOutputStream();
System.setOut(new PrintStream(bout, true, "UTF-8"));
oldSystemErr = System.err;
berr = new ByteArrayOutputStream();
System.setErr(new PrintStream(berr, true, "UTF-8"));
}
@After
public void tearDown() throws Exception {
super.tearDown();
System.setOut(oldSystemOut);
System.setErr(oldSystemErr);
}
@Test
public void testArgsParserTypicalUse() {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--morphline-id", "morphline_xyz",
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--mappers", "10",
"--reducers", "9",
"--fanout", "8",
"--max-segments", "7",
"--shards", "1",
"--verbose",
"file:///home",
"file:///dev",
};
Integer res = parser.parseArgs(args, conf, opts);
assertNull(res != null ? res.toString() : "", res);
assertEquals(Collections.singletonList(new Path("file:///tmp")), opts.inputLists);
assertEquals(new Path("file:/tmp/foo"), opts.outputDir);
assertEquals(new File(MINIMR_INSTANCE_DIR.getPath()), opts.solrHomeDir);
assertEquals(10, opts.mappers);
assertEquals(9, opts.reducers);
assertEquals(8, opts.fanout);
assertEquals(7, opts.maxSegments);
assertEquals(new Integer(1), opts.shards);
assertEquals(null, opts.fairSchedulerPool);
assertTrue(opts.isVerbose);
assertEquals(Arrays.asList(new Path("file:///home"), new Path("file:///dev")), opts.inputFiles);
assertEquals(RetainMostRecentUpdateConflictResolver.class.getName(), opts.updateConflictResolver);
assertEquals(MORPHLINE_FILE, opts.morphlineFile.getPath());
assertEquals("morphline_xyz", opts.morphlineId);
assertEmptySystemErrAndEmptySystemOut();
}
@Test
public void testArgsParserMultipleSpecsOfSameKind() {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--input-list", "file:///",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shards", "1",
"file:///home",
"file:///dev",
};
assertNull(parser.parseArgs(args, conf, opts));
assertEquals(Arrays.asList(new Path("file:///tmp"), new Path("file:///")), opts.inputLists);
assertEquals(Arrays.asList(new Path("file:///home"), new Path("file:///dev")), opts.inputFiles);
assertEquals(new Path("file:/tmp/foo"), opts.outputDir);
assertEquals(new File(MINIMR_INSTANCE_DIR.getPath()), opts.solrHomeDir);
assertEmptySystemErrAndEmptySystemOut();
}
@Test
public void testArgsParserTypicalUseWithEqualsSign() {
String[] args = new String[] {
"--input-list=file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir=file:/tmp/foo",
"--solr-home-dir=" + MINIMR_INSTANCE_DIR.getPath(),
"--mappers=10",
"--shards", "1",
"--verbose",
"file:///home",
"file:///dev",
};
assertNull(parser.parseArgs(args, conf, opts));
assertEquals(Collections.singletonList(new Path("file:///tmp")), opts.inputLists);
assertEquals(new Path("file:/tmp/foo"), opts.outputDir);
assertEquals(new File(MINIMR_INSTANCE_DIR.getPath()), opts.solrHomeDir);
assertEquals(10, opts.mappers);
assertEquals(new Integer(1), opts.shards);
assertEquals(null, opts.fairSchedulerPool);
assertTrue(opts.isVerbose);
assertEquals(Arrays.asList(new Path("file:///home"), new Path("file:///dev")), opts.inputFiles);
assertEmptySystemErrAndEmptySystemOut();
}
@Test
public void testArgsParserMultipleSpecsOfSameKindWithEqualsSign() {
String[] args = new String[] {
"--input-list=file:///tmp",
"--input-list=file:///",
"--morphline-file", MORPHLINE_FILE,
"--output-dir=file:/tmp/foo",
"--solr-home-dir=" + MINIMR_INSTANCE_DIR.getPath(),
"--shards", "1",
"file:///home",
"file:///dev",
};
assertNull(parser.parseArgs(args, conf, opts));
assertEquals(Arrays.asList(new Path("file:///tmp"), new Path("file:///")), opts.inputLists);
assertEquals(Arrays.asList(new Path("file:///home"), new Path("file:///dev")), opts.inputFiles);
assertEquals(new Path("file:/tmp/foo"), opts.outputDir);
assertEquals(new File(MINIMR_INSTANCE_DIR.getPath()), opts.solrHomeDir);
assertEmptySystemErrAndEmptySystemOut();
}
@Test
public void testArgsParserHelp() throws UnsupportedEncodingException {
String[] args = new String[] { "--help" };
assertEquals(new Integer(0), parser.parseArgs(args, conf, opts));
String helpText = new String(bout.toByteArray(), StandardCharsets.UTF_8);
assertTrue(helpText.contains("MapReduce batch job driver that "));
assertTrue(helpText.contains("bin/hadoop command"));
assertEquals(0, berr.toByteArray().length);
}
@Test
public void testArgsParserOk() {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shards", "1",
};
assertNull(parser.parseArgs(args, conf, opts));
assertEquals(new Integer(1), opts.shards);
assertEmptySystemErrAndEmptySystemOut();
}
@Test
public void testArgsParserUpdateConflictResolver() {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shards", "1",
"--update-conflict-resolver", NoChangeUpdateConflictResolver.class.getName(),
};
assertNull(parser.parseArgs(args, conf, opts));
assertEquals(NoChangeUpdateConflictResolver.class.getName(), opts.updateConflictResolver);
assertEmptySystemErrAndEmptySystemOut();
}
@Test
public void testArgsParserUnknownArgName() throws Exception {
String[] args = new String[] {
"--xxxxxxxxinputlist", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shards", "1",
};
assertArgumentParserException(args);
}
@Test
public void testArgsParserFileNotFound1() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/fileNotFound/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shards", "1",
};
assertArgumentParserException(args);
}
@Test
public void testArgsParserFileNotFound2() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", "/fileNotFound",
"--shards", "1",
};
assertArgumentParserException(args);
}
@Test
public void testArgsParserIntOutOfRange() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shards", "1",
"--mappers", "-20"
};
assertArgumentParserException(args);
}
@Test
public void testArgsParserIllegalFanout() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shards", "1",
"--fanout", "1" // must be >= 2
};
assertArgumentParserException(args);
}
@Test
public void testArgsParserSolrHomeMustContainSolrConfigFile() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--shards", "1",
"--solr-home-dir", "/",
};
assertArgumentParserException(args);
}
@Test
public void testArgsShardUrlOk() {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shard-url", "http://localhost:8983/solr/collection1",
"--shard-url", "http://localhost:8983/solr/collection2",
};
assertNull(parser.parseArgs(args, conf, opts));
assertEquals(Arrays.asList(
Collections.singletonList("http://localhost:8983/solr/collection1"),
Collections.singletonList("http://localhost:8983/solr/collection2")),
opts.shardUrls);
assertEquals(new Integer(2), opts.shards);
assertEmptySystemErrAndEmptySystemOut();
}
@Test
public void testArgsShardUrlMustHaveAParam() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shard-url",
};
assertArgumentParserException(args);
}
@Test
public void testArgsShardUrlAndShardsSucceeds() {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shards", "1",
"--shard-url", "http://localhost:8983/solr/collection1",
};
assertNull(parser.parseArgs(args, conf, opts));
assertEmptySystemErrAndEmptySystemOut();
}
@Test
public void testArgsShardUrlNoGoLive() {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shard-url", "http://localhost:8983/solr/collection1"
};
assertNull(parser.parseArgs(args, conf, opts));
assertEmptySystemErrAndEmptySystemOut();
assertEquals(new Integer(1), opts.shards);
}
@Test
public void testArgsShardUrlsAndZkhostAreMutuallyExclusive() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shard-url", "http://localhost:8983/solr/collection1",
"--shard-url", "http://localhost:8983/solr/collection1",
"--zk-host", "http://localhost:2185",
"--go-live"
};
assertArgumentParserException(args);
}
@Test
public void testArgsGoLiveAndSolrUrl() {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--shard-url", "http://localhost:8983/solr/collection1",
"--shard-url", "http://localhost:8983/solr/collection1",
"--go-live"
};
Integer result = parser.parseArgs(args, conf, opts);
assertNull(result);
assertEmptySystemErrAndEmptySystemOut();
}
@Test
public void testArgsZkHostNoGoLive() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--zk-host", "http://localhost:2185",
};
assertArgumentParserException(args);
}
@Test
public void testArgsGoLiveZkHostNoCollection() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--zk-host", "http://localhost:2185",
"--go-live"
};
assertArgumentParserException(args);
}
@Test
public void testArgsGoLiveNoZkHostOrSolrUrl() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(),
"--go-live"
};
assertArgumentParserException(args);
}
@Test
public void testNoSolrHomeDirOrZKHost() throws Exception {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--shards", "1",
};
assertArgumentParserException(args);
}
@Test
public void testZKHostNoSolrHomeDirOk() {
String[] args = new String[] {
"--input-list", "file:///tmp",
"--morphline-file", MORPHLINE_FILE,
"--output-dir", "file:/tmp/foo",
"--zk-host", "http://localhost:2185",
"--collection", "collection1",
};
assertNull(parser.parseArgs(args, conf, opts));
assertEmptySystemErrAndEmptySystemOut();
}
private void assertEmptySystemErrAndEmptySystemOut() {
assertEquals(0, bout.toByteArray().length);
assertEquals(0, berr.toByteArray().length);
}
private void assertArgumentParserException(String[] args) throws UnsupportedEncodingException {
assertEquals("should have returned fail code", new Integer(1), parser.parseArgs(args, conf, opts));
assertEquals("no sys out expected:" + new String(bout.toByteArray(), StandardCharsets.UTF_8), 0, bout.toByteArray().length);
String usageText;
usageText = new String(berr.toByteArray(), StandardCharsets.UTF_8);
assertTrue("should start with usage msg \"usage: hadoop \":" + usageText, usageText.startsWith("usage: hadoop "));
}
}