/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.falcon.entity;
import org.apache.falcon.entity.parser.ClusterEntityParser;
import org.apache.falcon.entity.parser.EntityParserFactory;
import org.apache.falcon.entity.parser.FeedEntityParser;
import org.apache.falcon.entity.store.ConfigurationStore;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.cluster.Interface;
import org.apache.falcon.entity.v0.cluster.Interfacetype;
import org.apache.falcon.entity.v0.feed.CatalogTable;
import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.entity.v0.feed.Location;
import org.apache.falcon.entity.v0.feed.LocationType;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.io.InputStream;
import java.util.List;
/**
* Test for storage factory methods in feed helper.
*/
public class StorageFactoryTest {
private static final String CLUSTER_XML = "/config/cluster/cluster-0.1.xml";
private static final String FS_FEED_UNIFORM = "/config/feed/feed-0.1.xml";
private static final String FS_FEED_OVERRIDE = "/config/feed/feed-0.2.xml";
private static final String TABLE_FEED_UNIFORM = "/config/feed/hive-table-feed.xml";
private static final String TABLE_FEED_OVERRIDE = "/config/feed/hive-table-feed-out.xml";
private static final String OVERRIDE_TBL_LOC = "/testCluster/clicks-summary/ds=${YEAR}-${MONTH}-${DAY}-${HOUR}";
private final ClusterEntityParser clusterParser =
(ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER);
private final FeedEntityParser feedParser =
(FeedEntityParser) EntityParserFactory.getParser(EntityType.FEED);
private Cluster clusterEntity;
private Feed fsFeedWithUniformStorage;
private Feed fsFeedWithOverriddenStorage;
private Feed tableFeedWithUniformStorage;
private Feed tableFeedWithOverriddenStorage;
@BeforeClass
public void setup() throws Exception {
InputStream stream = this.getClass().getResourceAsStream(CLUSTER_XML);
clusterEntity = clusterParser.parse(stream);
stream.close();
Interface registry = ClusterHelper.getInterface(clusterEntity, Interfacetype.REGISTRY);
registry.setEndpoint("thrift://localhost:9083");
ConfigurationStore.get().publish(EntityType.CLUSTER, clusterEntity);
stream = this.getClass().getResourceAsStream(FS_FEED_UNIFORM);
fsFeedWithUniformStorage = feedParser.parse(stream);
stream.close();
stream = this.getClass().getResourceAsStream(FS_FEED_OVERRIDE);
fsFeedWithOverriddenStorage = feedParser.parse(stream);
stream.close();
stream = this.getClass().getResourceAsStream(TABLE_FEED_UNIFORM);
tableFeedWithUniformStorage = feedParser.parse(stream);
stream.close();
stream = this.getClass().getResourceAsStream(TABLE_FEED_OVERRIDE);
tableFeedWithOverriddenStorage = feedParser.parse(stream);
stream.close();
}
@AfterClass
public void tearDown() throws Exception {
ConfigurationStore.get().remove(EntityType.CLUSTER, clusterEntity.getName());
}
@DataProvider (name = "locationsDataProvider")
private Object[][] createLocationsDataProvider() {
return new Object[][] {
{fsFeedWithUniformStorage, "/projects/falcon/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}"},
{fsFeedWithOverriddenStorage, "/testCluster/projects/falcon/clicks"},
};
}
@Test (dataProvider = "locationsDataProvider")
public void testGetLocations(Feed feed, String dataPath) {
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
FeedHelper.getCluster(feed, clusterEntity.getName());
List<Location> locations = FeedHelper.getLocations(feedCluster, feed);
for (Location location : locations) {
if (location.getType() == LocationType.DATA) {
Assert.assertEquals(location.getPath(), dataPath);
}
}
}
@DataProvider (name = "tableDataProvider")
private Object[][] createTableDataProvider() {
return new Object[][] {
{tableFeedWithUniformStorage, "catalog:default:clicks#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}"},
{tableFeedWithOverriddenStorage, "catalog:testCluster:clicks-summary#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}"},
};
}
@Test (dataProvider = "tableDataProvider")
public void testGetTable(Feed feed, String dataPath) {
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
FeedHelper.getCluster(feed, clusterEntity.getName());
CatalogTable table = FeedHelper.getTable(feedCluster, feed);
Assert.assertEquals(table.getUri(), dataPath);
}
private static final String UNIFORM_TABLE = "${hcatNode}/default/clicks/ds=${YEAR}-${MONTH}-${DAY}-${HOUR}";
private static final String OVERRIDETBL = "${hcatNode}/default/clicks-summary/ds=${YEAR}-${MONTH}-${DAY}-${HOUR}";
@DataProvider (name = "uniformFeedStorageDataProvider")
private Object[][] createUniformFeedStorageDataProvider() {
return new Object[][] {
{fsFeedWithUniformStorage, Storage.TYPE.FILESYSTEM,
"${nameNode}/projects/falcon/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}", },
{fsFeedWithOverriddenStorage, Storage.TYPE.FILESYSTEM, "${nameNode}/projects/falcon/clicks"},
{tableFeedWithUniformStorage, Storage.TYPE.TABLE, UNIFORM_TABLE},
{tableFeedWithOverriddenStorage, Storage.TYPE.TABLE, OVERRIDETBL},
};
}
@Test (dataProvider = "uniformFeedStorageDataProvider")
public void testCreateStorageWithFeed(Feed feed, Storage.TYPE storageType,
String dataLocation) throws Exception {
Storage storage = FeedHelper.createStorage(feed);
Assert.assertEquals(storage.getType(), storageType);
Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), dataLocation);
if (storageType == Storage.TYPE.TABLE) {
Assert.assertEquals(((CatalogStorage) storage).getDatabase(), "default");
}
}
@DataProvider (name = "overriddenFeedStorageDataProvider")
private Object[][] createFeedStorageDataProvider() {
return new Object[][] {
{fsFeedWithUniformStorage, Storage.TYPE.FILESYSTEM,
"/projects/falcon/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}", },
{fsFeedWithOverriddenStorage, Storage.TYPE.FILESYSTEM, "/testCluster/projects/falcon/clicks"},
{tableFeedWithUniformStorage, Storage.TYPE.TABLE, "/default/clicks/ds=${YEAR}-${MONTH}-${DAY}-${HOUR}"},
{tableFeedWithOverriddenStorage, Storage.TYPE.TABLE, OVERRIDE_TBL_LOC},
};
}
@Test (dataProvider = "overriddenFeedStorageDataProvider")
public void testCreateStorageWithFeedAndClusterEntity(Feed feed, Storage.TYPE storageType,
String dataLocation) throws Exception {
Storage storage = FeedHelper.createStorage(clusterEntity, feed);
Assert.assertEquals(storage.getType(), storageType);
if (storageType == Storage.TYPE.FILESYSTEM) {
dataLocation = ClusterHelper.getStorageUrl(clusterEntity) + dataLocation;
} else if (storageType == Storage.TYPE.TABLE) {
dataLocation =
ClusterHelper.getInterface(clusterEntity, Interfacetype.REGISTRY).getEndpoint() + dataLocation;
}
Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), dataLocation);
}
@Test (dataProvider = "overriddenFeedStorageDataProvider")
public void testCreateStorageWithFeedAndClusterName(Feed feed, Storage.TYPE storageType,
String dataLocation) throws Exception {
Storage storage = FeedHelper.createStorage(clusterEntity.getName(), feed);
Assert.assertEquals(storage.getType(), storageType);
if (storageType == Storage.TYPE.FILESYSTEM) {
dataLocation = ClusterHelper.getStorageUrl(clusterEntity) + dataLocation;
} else if (storageType == Storage.TYPE.TABLE) {
dataLocation =
ClusterHelper.getInterface(clusterEntity, Interfacetype.REGISTRY).getEndpoint() + dataLocation;
}
Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), dataLocation);
}
@Test (dataProvider = "overriddenFeedStorageDataProvider")
public void testCreateStorageWithFeedAndFeedCluster(Feed feed, Storage.TYPE storageType,
String dataLocation) throws Exception {
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
FeedHelper.getCluster(feed, clusterEntity.getName());
Storage storage = FeedHelper.createStorage(feedCluster, feed);
Assert.assertEquals(storage.getType(), storageType);
if (storageType == Storage.TYPE.FILESYSTEM) {
dataLocation = ClusterHelper.getStorageUrl(clusterEntity) + dataLocation;
} else if (storageType == Storage.TYPE.TABLE) {
dataLocation =
ClusterHelper.getInterface(clusterEntity, Interfacetype.REGISTRY).getEndpoint() + dataLocation;
}
Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), dataLocation);
}
@Test (dataProvider = "overriddenFeedStorageDataProvider")
public void testCreateStorageWithAll(Feed feed, Storage.TYPE storageType,
String dataLocation) throws Exception {
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
FeedHelper.getCluster(feed, clusterEntity.getName());
Storage storage = FeedHelper.createStorage(feedCluster, feed, clusterEntity);
Assert.assertEquals(storage.getType(), storageType);
if (storageType == Storage.TYPE.FILESYSTEM) {
dataLocation = ClusterHelper.getStorageUrl(clusterEntity) + dataLocation;
} else if (storageType == Storage.TYPE.TABLE) {
dataLocation =
ClusterHelper.getInterface(clusterEntity, Interfacetype.REGISTRY).getEndpoint() + dataLocation;
}
Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), dataLocation);
}
@Test (dataProvider = "overriddenFeedStorageDataProvider")
public void testCreateReadOnlyStorage(Feed feed, Storage.TYPE storageType,
String dataLocation) throws Exception {
Storage readOnlyStorage = FeedHelper.createReadOnlyStorage(clusterEntity, feed);
Assert.assertEquals(readOnlyStorage.getType(), storageType);
if (storageType == Storage.TYPE.FILESYSTEM) {
dataLocation = ClusterHelper.getReadOnlyStorageUrl(clusterEntity) + dataLocation;
} else if (storageType == Storage.TYPE.TABLE) {
dataLocation =
ClusterHelper.getInterface(clusterEntity, Interfacetype.REGISTRY).getEndpoint() + dataLocation;
}
Assert.assertEquals(readOnlyStorage.getUriTemplate(LocationType.DATA), dataLocation);
}
@DataProvider (name = "uriTemplateDataProvider")
private Object[][] createUriTemplateDataProvider() {
return new Object[][] {
{Storage.TYPE.FILESYSTEM, "/projects/falcon/clicks"},
{Storage.TYPE.FILESYSTEM, "/testCluster/projects/falcon/clicks"},
{Storage.TYPE.TABLE, "/default/clicks/ds=${YEAR}-${MONTH}-${DAY}-${HOUR}"},
{Storage.TYPE.TABLE, OVERRIDE_TBL_LOC},
};
}
@Test (dataProvider = "uriTemplateDataProvider")
public void testCreateStorageWithUriTemplate(Storage.TYPE storageType,
String dataLocation) throws Exception {
String uriTemplate = null;
if (storageType == Storage.TYPE.FILESYSTEM) {
uriTemplate = "DATA=" + ClusterHelper.getStorageUrl(clusterEntity) + dataLocation + "#";
dataLocation = ClusterHelper.getStorageUrl(clusterEntity) + dataLocation;
} else if (storageType == Storage.TYPE.TABLE) {
uriTemplate =
ClusterHelper.getInterface(clusterEntity, Interfacetype.REGISTRY).getEndpoint() + dataLocation;
dataLocation = uriTemplate;
}
Storage storage = FeedHelper.createStorage(storageType.name(), uriTemplate);
Assert.assertEquals(storage.getType(), storageType);
Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), dataLocation);
}
@DataProvider (name = "storageTypeDataProvider")
private Object[][] createStorageTypeDataProvider() {
return new Object[][] {
{fsFeedWithUniformStorage, Storage.TYPE.FILESYSTEM},
{fsFeedWithOverriddenStorage, Storage.TYPE.FILESYSTEM},
{tableFeedWithUniformStorage, Storage.TYPE.TABLE},
{tableFeedWithOverriddenStorage, Storage.TYPE.TABLE},
};
}
@Test (dataProvider = "storageTypeDataProvider")
public void testGetStorageTypeWithFeed(Feed feed, Storage.TYPE expectedStorageType) throws Exception {
Storage.TYPE actualStorageType = FeedHelper.getStorageType(feed);
Assert.assertEquals(actualStorageType, expectedStorageType);
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
FeedHelper.getCluster(feed, clusterEntity.getName());
actualStorageType = FeedHelper.getStorageType(feed, feedCluster);
Assert.assertEquals(actualStorageType, expectedStorageType);
actualStorageType = FeedHelper.getStorageType(feed, clusterEntity);
Assert.assertEquals(actualStorageType, expectedStorageType);
}
}