/*
* Copyright 2011-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.data.hadoop.fs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.net.URI;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.data.hadoop.TestUtils;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Costin Leau
*/
@SuppressWarnings("deprecation")
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
public class DistributedCacheTest {
@Autowired
Configuration cfg;
@Autowired
FileSystem fs;
@Autowired
ApplicationContext ctx;
@Before
public void init() throws Exception {
TestUtils.writeToFS(cfg, "local/" + UUID.randomUUID() + ".txt");
TestUtils.writeToFS(cfg, "local/" + UUID.randomUUID() + ".txt");
TestUtils.writeToFS(cfg, "local/" + UUID.randomUUID() + ".txt");
HdfsResourceLoader loader = new HdfsResourceLoader(cfg);
assertTrue(loader.getResources("~/local/*.txt").length >= 3);
loader.close();
ctx.getBean("hadoopCache");
}
@After
public void destroy() throws Exception {
FileSystem fs = FileSystem.get(cfg);
fs.delete(new Path("local/"), true);
}
// we do extra parsing since the classpath url behaves different on cloudera then Apache Vanilla
@Test
public void testClassPathArchives() throws Exception {
if (System.getProperty("os.name").toLowerCase().startsWith("win")) {
System.setProperty("path.separator", ":");
}
Path[] archives = DistributedCache.getArchiveClassPaths(cfg);
assertEquals(2, archives.length);
assertEquals(new Path("/cp/some-zip.zip"), archives[0]);
}
// we do extra parsing since the classpath url behaves different on cloudera then Apache Vanilla
@Test
public void testClassPathFiles() throws Exception {
Path[] files = DistributedCache.getFileClassPaths(cfg);
assertEquals(1, files.length);
Path path = files[0];
String p = path.toUri().getPath();
// remove fragment
int index = p.indexOf("#");
if (index >= 0) {
p = p.substring(0, index);
}
assertEquals("/cp/some-library.jar", p);
}
@Test
public void testCacheArchives() throws Exception {
URI[] archives = DistributedCache.getCacheArchives(cfg);
System.out.println(Arrays.toString(archives));
assertEquals(3, archives.length);
assertEquals("/cp/some-zip.zip", archives[0].getPath());
//assertEquals("some-zip.zip", archives[0].getFragment());
assertEquals("/cp/some-extra-zip.zip", archives[1].getPath());
assertEquals("/cache/some-archive.tgz", archives[2].getPath());
//assertEquals("main-archive", archives[1].getFragment());
}
@Test
public void testCacheFiles() throws Exception {
URI[] files = DistributedCache.getCacheFiles(cfg);
System.out.println(Arrays.toString(files));
assertEquals(2, files.length);
assertEquals("/cp/some-library.jar", files[0].getPath());
assertEquals("library.jar", files[0].getFragment());
assertEquals("/cache/some-resource.res", files[1].getPath());
assertEquals("some-resource.res", files[1].getFragment());
}
@Test
public void testMisc() throws Exception {
assertTrue(ctx.containsBean("hadoopCache"));
assertTrue(DistributedCache.getSymlink(cfg));
}
@Test
public void testClassPathFragment() throws Exception {
DistributedCacheFactoryBean dcache = new DistributedCacheFactoryBean();
dcache.setConfiguration(cfg);
dcache.setCreateSymlink(false);
Set<DistributedCacheFactoryBean.CacheEntry> jars = new HashSet<DistributedCacheFactoryBean.CacheEntry>();
jars.add(new DistributedCacheFactoryBean.CacheEntry(DistributedCacheFactoryBean.CacheEntry.EntryType.CP,
"/cp/some-library-1.jar"));
dcache.setEntries(jars);
dcache.afterPropertiesSet();
}
}