package com.thinkbiganalytics.nifi.core.api.metadata;
/*-
* #%L
* thinkbig-nifi-core-service-api
* %%
* Copyright (C) 2017 ThinkBig Analytics
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.thinkbiganalytics.metadata.api.op.FeedDependencyDeltaResults;
import com.thinkbiganalytics.metadata.rest.model.data.Datasource;
import com.thinkbiganalytics.metadata.rest.model.data.DirectoryDatasource;
import com.thinkbiganalytics.metadata.rest.model.data.HiveTableDatasource;
import com.thinkbiganalytics.metadata.rest.model.feed.Feed;
import com.thinkbiganalytics.metadata.rest.model.feed.FeedDestination;
import com.thinkbiganalytics.metadata.rest.model.op.DataOperation;
import com.thinkbiganalytics.metadata.rest.model.op.DataOperation.State;
import com.thinkbiganalytics.metadata.rest.model.op.Dataset;
import com.thinkbiganalytics.metadata.rest.model.op.HiveTablePartitions;
import com.thinkbiganalytics.metadata.sla.api.Metric;
import org.joda.time.DateTime;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Optional;
import java.util.Properties;
import javax.annotation.Nonnull;
/**
* Provides services for working with feeds and data sources
*/
public interface MetadataProvider {
/**
* gets the id for the feed given
*
* @param category the category containing the feed
* @param feedName the name of the feed
* @return a feed id
*/
String getFeedId(String category, String feedName);
/**
* returns the api service class used to obtain information about feed deltas
*
* @param feedId the feed in question
* @return a service object to get more information about feed deltas
*/
FeedDependencyDeltaResults getFeedDependentResultDeltas(String feedId);
/**
* Ensures the feed exists in the metadata store. Finds an existing feed of if none exists then one is created.
*
* @param categoryName The category for the feed
* @param feedName The name of the feed
* @param description A description to associate with the feed
* @return the feed
*/
Feed ensureFeed(String categoryName, String feedName, String description);
/**
* gets the data source service by the given name
*
* @param dsName the data source name
* @return the data source description
*/
Datasource getDatasourceByName(String dsName);
/**
* ensures the feed source given by the feedId and datasourceId
*
* @param feedId the feed id
* @param datasourceId the datasource id
* @return the feed
*/
Feed ensureFeedSource(String feedId, String datasourceId);
/**
* ensures the feed destination given by the feedId and datasourceId
*
* @param feedId the feed id
* @param datasourceId the datasource id
* @return the feed
*/
Feed ensureFeedDestination(String feedId, String datasourceId);
/**
* gets, or creates a feed, based on certain preconditions given as Metric's
*
* @param feedId the feed id
* @param metrics the conditions to be met
* @return the feed
*/
Feed ensurePrecondition(String feedId, Metric... metrics);
/**
* merge the given properties to the current feed properties
*
* @param feedId the feed id
* @param props the datasource id
* @return the updated properties
*/
Properties updateFeedProperties(String feedId, Properties props);
/**
* gets, or creates, a datasource
*
* @param datasetName the dataset name
* @param descr A description for the datasource
* @param path the path for the data
* @return a directory data source that can be used to fetch or modify metadata
*/
DirectoryDatasource ensureDirectoryDatasource(String datasetName, String descr, Path path);
/**
* gets, or creates, a table in hive to use a data source
*
* @param datasetName the dataset name
* @param descr a description for the data source
* @param databaseName the database name where the table (will) reside(s)
* @param tableName the name of the table
* @return a HiveTableDatasource that can be used to fetch or modify table metadata
*/
HiveTableDatasource ensureHiveTableDatasource(String datasetName, String descr, String databaseName, String tableName);
/**
* creates a file system Dataset at one or more paths
*
* @param dds the directory datasource which describes the locations of the data
* @param paths one or more paths
* @return a dataset object that is used to fetch or modify meta data
*/
Dataset createDataset(DirectoryDatasource dds, Path... paths);
/**
* create a data set that can be used to describe the metadata for a directory data source and it's paths
*
* @param dds the directory datasource which describes the dataset
* @param paths one or more paths
* @return a dataset object that is used to fetch or modify meta data
*/
Dataset createDataset(DirectoryDatasource dds, ArrayList<Path> paths);
/**
* create a data set that can be used to describe the metadata for a hive data source and it's partitions
*
* @param hds the datasource that describes the hive table
* @param partitions the partitions of the table
* @return a dataset object that is used to fetch or modify meta data
*/
Dataset createDataset(HiveTableDatasource hds, HiveTablePartitions partitions);
/**
* Begin tracking the run time of the operation on the feedDestination
*
* @param feedDestination the feed destination
* @param opStart the time it starts
* @return a data operation
*/
DataOperation beginOperation(FeedDestination feedDestination, DateTime opStart);
/**
* complete the operation begun by beginOperation
*
* @param id the id of the operation
* @param status the status of the operation
* @param changeSet the changeset resulting from the operation
* @return a data operation object which is used to track the metadata of the operation
*/
DataOperation completeOperation(String id, String status, Dataset changeSet);
/**
* complete the operation begun by beginOperation
*
* @param id the id of the operation
* @param status the status of the operation
* @param state the state of the operation
* @return a data operation object which is used to track the metadata of the operation
*/
DataOperation completeOperation(String id, String status, State state);
/**
* Gets the properties of the specified feed.
*
* @param id the feed id
* @return the properties
*/
Properties getFeedProperties(@Nonnull final String id);
/**
* Merges the specified properties into the feed's properties.
*
* @param id the feed id
* @param props the new properties
* @return the merged properties
*/
Properties mergeFeedProperties(@Nonnull final String id, @Nonnull final Properties props);
/**
* Gets the data source with the specified id.
*
* @param id the data source id
* @return the data source, if found
*/
Optional<Datasource> getDatasource(@Nonnull String id);
}