/**
* DataCleaner (community edition)
* Copyright (C) 2014 Neopost - Customer Information Management
*
* This copyrighted material is made available to anyone wishing to use, modify,
* copy, or redistribute it subject to the terms and conditions of the GNU
* Lesser General Public License, as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this distribution; if not, write to:
* Free Software Foundation, Inc.
* 51 Franklin Street, Fifth Floor
* Boston, MA 02110-1301 USA
*/
package org.datacleaner.spark;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.metamodel.util.Resource;
import org.datacleaner.configuration.DataCleanerConfigurationImpl;
import org.datacleaner.spark.utils.HdfsHelper;
import org.datacleaner.util.HadoopResource;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class SparkConfigurationReaderInterceptorTest {
@Before
public void setup() {
// setup code needed to register the "last known" hadoop configuration
// in HdfsHelper.
new HdfsHelper(new Configuration());
}
@After
public void tearDown() {
HdfsHelper.clear();
}
@Test
public void testRemoveServerFromHdfsUrl() throws Exception {
final SparkConfigurationReaderInterceptor interceptor =
new SparkConfigurationReaderInterceptor(new HashMap<>());
final Resource resource = interceptor
.createResource("hdfs://{foo.bar.hadoop.environment}/datacleaner/data/companies_1.csv",
new DataCleanerConfigurationImpl());
assertTrue("Found: " + resource.getClass().getName(), resource instanceof HadoopResource);
assertEquals("hdfs:///datacleaner/data/companies_1.csv", resource.getQualifiedPath());
}
@Test
public void testSimpleHdfsUrl() throws Exception {
final SparkConfigurationReaderInterceptor interceptor =
new SparkConfigurationReaderInterceptor(new HashMap<>());
final Resource resource = interceptor
.createResource("hdfs:///datacleaner/data/companies_2.csv", new DataCleanerConfigurationImpl());
assertTrue("Found: " + resource.getClass().getName(), resource instanceof HadoopResource);
assertEquals("hdfs:///datacleaner/data/companies_2.csv", resource.getQualifiedPath());
}
@Test
public void testSimpleSchemeLessUrl() throws Exception {
final SparkConfigurationReaderInterceptor interceptor =
new SparkConfigurationReaderInterceptor(new HashMap<>());
final Resource resource =
interceptor.createResource("/datacleaner/data/companies_3.csv", new DataCleanerConfigurationImpl());
assertTrue("Found: " + resource.getClass().getName(), resource instanceof HadoopResource);
assertEquals("hdfs:///datacleaner/data/companies_3.csv", resource.getQualifiedPath());
}
}