HdfsResourceLoaderLegacyTest.java example

Explorer
spring-hadoop-master
/*
 * Copyright 2011 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.springframework.data.hadoop.fs;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URL;
import java.util.Arrays;
import java.util.UUID;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.DecompressorStream;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.data.hadoop.TestUtils;
import org.springframework.data.hadoop.configuration.ConfigurationFactoryBean;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

/**
 * Test for interacting with Hadoop HDFS.
 *
 * @author Costin Leau
 */
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration("/org/springframework/data/hadoop/hadoop-ctx.xml")
public class HdfsResourceLoaderLegacyTest {

	@Autowired
	private Configuration cfg;
	private FileSystem fs;
	private HdfsResourceLoader loader;

	@Before
	public void before() throws Exception {
		FileSystemFactoryBean fsf = new FileSystemFactoryBean();
		fsf.setConfiguration(cfg);
		fsf.afterPropertiesSet();

		fs = fsf.getObject();

		System.out.println("Current user is " + UserGroupInformation.getCurrentUser());
		System.out.println("Home dir is " + fs.getHomeDirectory().toString());

		loader = new HdfsResourceLoader(cfg, null);
	}

	@After
	public void after() throws Exception {
		if (fs != null) {
			fs.close();
		}
	}

	@Test
	public void testPatternResolver() throws Exception {
		Resource[] resources = loader.getResources("**/*");
		System.out.println(resources.length);
		System.out.println(Arrays.toString(resources));

		Resource[] homeResources = loader.getResources("~/**/*");
		System.out.println(homeResources.length);
		System.out.println(Arrays.toString(resources));
		assertArrayEquals(resources, homeResources);

		Resource[] res = loader.getResources("/**/user/**/*");
		System.out.println(Arrays.toString(res));

		System.out.println(Arrays.toString(loader.getResources("/**/user/**/*.xml")));
	}

	@Test
	public void testConnectExistingFS() throws Exception {
		System.out.println(fs.toString());
		System.out.println(fs.getUri());
		System.out.println(fs.getHomeDirectory());
		FileStatus[] listStatus = fs.listStatus(new Path("/"));

		for (FileStatus fileStatus : listStatus) {
			System.out.println(fileStatus.getPath());
		}

		Resource resource = loader.getResource("/");
		System.out.println(resource.isReadable());
		System.out.println(resource.contentLength());
	}

	@Test
	public void testWriteable() throws Exception {
		String name = "test-" + UUID.randomUUID() + ".file";
		Path path = new Path(name);

		try {

			Resource resource = loader.getResource(name);
			System.out.println(resource.toString());
			assertFalse(resource.exists());
			assertFalse(resource.isReadable());
			assertFalse(resource.isOpen());

			resource = TestUtils.writeToFS(loader, name);
			assertTrue(resource.exists());
			assertTrue(resource.isReadable());
			assertTrue(resource.isOpen());

			InputStream in = resource.getInputStream();

			byte[] copy = new byte[name.length()];
			in.read(copy);
			in.close();
			assertArrayEquals(name.getBytes(), copy);
		} finally {
			fs.delete(path, true);
		}
	}

	@SuppressWarnings("deprecation")
	@Test
	public void testResolve() throws Exception {
		Resource resource = loader.getResource("/test");
		try {
			URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(fs.getConf()));
		} catch (Error err) {
			//somebody already registered an URL handler...
			// get out
		}

		System.out.println(((HdfsResource) resource).getPath().makeQualified(fs));
		System.out.println(resource.getURI());

		resource = loader.getResource("test");
		System.out.println(resource.getURI());
		System.out.println(resource.getURL());
	}

	@Test
	public void testPathWithFragment() throws Exception {
		String name = "test-" + UUID.randomUUID() + ".file#fragment";

		Path path = new Path(name);
		Resource resource = loader.getResource(name);

		try {
			System.out.println(resource.toString());
			assertFalse(resource.exists());
			assertFalse(resource.isReadable());
			assertFalse(resource.isOpen());

			resource = TestUtils.writeToFS(loader, resource.getURI().toString());
			//			assertTrue(resource instanceof WritableResource);
			//			WritableResource wr = (WritableResource) resource;
			//			assertTrue(wr.isWritable());
			//
			//			byte[] bytes = name.getBytes();
			//			OutputStream out = wr.getOutputStream();
			//
			//			out.write(bytes);
			//			out.close();

			assertTrue(resource.exists());
			assertTrue(resource.isReadable());
			assertTrue(resource.isOpen());

			URI uri = resource.getURI();
			assertEquals("fragment", uri.getFragment());

		} finally {
			fs.delete(path, true);
		}

	}

	@Test
	public void testFilesMatch() throws Exception {
		try {
			TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt");
			TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt");
			TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt");

			Resource[] resources = loader.getResources("local/*.txt");
			assertTrue(resources.length >= 3);
		} finally {
			fs.delete(new Path("local/"), true);
		}
	}

	@Test
	public void testFilesMatchWithHomePrefix() throws Exception {
		try {
			TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt");
			TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt");
			TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt");

			Resource[] resources = loader.getResources("~/local/*.txt");
			assertTrue(resources.length >= 3);
		} finally {
			fs.delete(new Path("local/"), true);
		}
	}

	@Test
	public void testFilesMatchWithPrefix() throws Exception {
		try {
			TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt");
			TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt");
			TestUtils.writeToFS(loader, "local/" + UUID.randomUUID() + ".txt");

			Resource[] resources = loader.getResources("./local/../local/*.txt");
			assertTrue(resources.length >= 3);
		} finally {
			fs.delete(new Path("local/"), true);
		}
	}

	@Test
	public void testDecompressedStream() throws Exception {
		DefaultCodec codec = new DefaultCodec();
		codec.setConf(fs.getConf());
		String name = "local/" + UUID.randomUUID() + codec.getDefaultExtension();
		OutputStream outputStream = codec.createOutputStream(fs.create(new Path(name)));
		byte[] content = name.getBytes();
		outputStream.write(content);
		outputStream.close();

		Resource resource = loader.getResource(name);
		assertNotNull(resource);
		InputStream inputStream = resource.getInputStream();
		assertEquals(DecompressorStream.class, inputStream.getClass());
		assertTrue(TestUtils.compareStreams(new ByteArrayInputStream(content), inputStream));
	}

	@Test
	public void testCompressedStream() throws Exception {

		DefaultCodec codec = new DefaultCodec();
		codec.setConf(fs.getConf());
		String name = "local/" + UUID.randomUUID() + codec.getDefaultExtension();
		OutputStream outputStream = codec.createOutputStream(fs.create(new Path(name)));
		byte[] content = name.getBytes();
		outputStream.write(content);
		outputStream.close();

		loader.setUseCodecs(false);

		try {
			Resource resource = loader.getResource(name);
			assertNotNull(resource);
			InputStream inputStream = resource.getInputStream();
			System.out.println(inputStream.getClass());
			assertFalse(DecompressorStream.class.equals(inputStream.getClass()));
			assertFalse(TestUtils.compareStreams(new ByteArrayInputStream(content), inputStream));
		} finally {
			loader.setUseCodecs(true);
		}
	}

	@Test
	public void testURLCycle() throws Exception {
		ConfigurationFactoryBean cfactory;
		cfactory = new ConfigurationFactoryBean();
		cfactory.setRegisterUrlHandler(true);
		cfactory.afterPropertiesSet();

		/* create hdfs resource loader */
		HdfsResourceLoader ldr = new HdfsResourceLoader(cfactory.getObject(), null, null);
		assertNotNull(ldr);
	}
}