DatasetAdminService.java

/*
 * Copyright © 2016 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.cdap.data2.datafabric.dataset.service.executor;

import co.cask.cdap.api.dataset.Dataset;
import co.cask.cdap.api.dataset.DatasetAdmin;
import co.cask.cdap.api.dataset.DatasetContext;
import co.cask.cdap.api.dataset.DatasetDefinition;
import co.cask.cdap.api.dataset.DatasetManagementException;
import co.cask.cdap.api.dataset.DatasetProperties;
import co.cask.cdap.api.dataset.DatasetSpecification;
import co.cask.cdap.common.BadRequestException;
import co.cask.cdap.common.NotFoundException;
import co.cask.cdap.common.conf.CConfiguration;
import co.cask.cdap.data.dataset.SystemDatasetInstantiator;
import co.cask.cdap.data.dataset.SystemDatasetInstantiatorFactory;
import co.cask.cdap.data2.datafabric.dataset.DatasetType;
import co.cask.cdap.data2.datafabric.dataset.RemoteDatasetFramework;
import co.cask.cdap.data2.datafabric.dataset.type.DatasetClassLoaderProvider;
import co.cask.cdap.data2.datafabric.dataset.type.DirectoryClassLoaderProvider;
import co.cask.cdap.data2.metadata.store.MetadataStore;
import co.cask.cdap.data2.metadata.system.DatasetSystemMetadataWriter;
import co.cask.cdap.data2.metadata.system.SystemMetadataWriter;
import co.cask.cdap.proto.DatasetTypeMeta;
import co.cask.cdap.proto.Id;
import com.google.inject.Inject;
import org.apache.twill.filesystem.LocationFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;

/**
 * Handles Dataset admin operations.
 */
public class DatasetAdminService {
  private static final Logger LOG = LoggerFactory.getLogger(DatasetAdminService.class);

  private final RemoteDatasetFramework dsFramework;
  private final CConfiguration cConf;
  private final LocationFactory locationFactory;
  private final SystemDatasetInstantiatorFactory datasetInstantiatorFactory;
  private final MetadataStore metadataStore;

  @Inject
  public DatasetAdminService(RemoteDatasetFramework dsFramework, CConfiguration cConf, LocationFactory locationFactory,
                             SystemDatasetInstantiatorFactory datasetInstantiatorFactory, MetadataStore metadataStore) {
    this.dsFramework = dsFramework;
    this.cConf = cConf;
    this.locationFactory = locationFactory;
    this.datasetInstantiatorFactory = datasetInstantiatorFactory;
    this.metadataStore = metadataStore;
  }

  public boolean exists(Id.DatasetInstance datasetInstanceId) throws Exception {
    DatasetAdmin datasetAdmin = getDatasetAdmin(datasetInstanceId);
    return datasetAdmin.exists();
  }

  /**
   * Configures and creates a Dataset
   *
   * @param datasetInstanceId dataset instance to be created
   * @param typeMeta type meta for the dataset
   * @param props dataset instance properties
   * @param existing true, if dataset already exists (in case of update)
   * @return dataset specification
   * @throws Exception
   */
  public DatasetSpecification create(Id.DatasetInstance datasetInstanceId, DatasetTypeMeta typeMeta,
                                     DatasetProperties props, boolean existing) throws Exception {
    LOG.info("Creating dataset instance {}, type meta: {}, props: {}", datasetInstanceId, typeMeta, props);
    try (DatasetClassLoaderProvider classLoaderProvider =
           new DirectoryClassLoaderProvider(cConf, locationFactory)) {
      DatasetType type = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);

      if (type == null) {
        String msg = String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta);
        LOG.error(msg);
        throw new BadRequestException(msg);
      }

      DatasetSpecification spec = type.configure(datasetInstanceId.getId(), props);
      DatasetContext context = DatasetContext.from(datasetInstanceId.getNamespaceId());
      DatasetAdmin admin = type.getAdmin(context, spec);
      try {
        admin.create();

        writeSystemMetadata(datasetInstanceId, spec, props, typeMeta, type, context, existing);

        return spec;
      } catch (IOException e) {
        String msg = String.format("Error creating dataset \"%s\": %s", datasetInstanceId, e.getMessage());
        LOG.error(msg, e);
        throw new IOException(msg, e);
      }
    } catch (IOException e) {
      String msg = String.format("Error instantiating the dataset admin for dataset %s", datasetInstanceId);
      LOG.error(msg, e);
      throw new IOException(msg, e);
    }
  }

  private void writeSystemMetadata(Id.DatasetInstance datasetInstanceId, DatasetSpecification spec,
                                   DatasetProperties props, DatasetTypeMeta typeMeta, DatasetType type,
                                   DatasetContext context, boolean existing) throws IOException {
    // add system metadata for user datasets only
    if (isUserDataset(datasetInstanceId)) {
      Dataset dataset = null;
      try {
        try {
          dataset = type.getDataset(context, spec, DatasetDefinition.NO_ARGUMENTS);
        } catch (Exception e) {
          LOG.warn("Exception while instantiating Dataset {}", datasetInstanceId, e);
        }

        // Make sure to write whatever system metadata that can be derived
        // even if the above instantiation throws exception
        SystemMetadataWriter systemMetadataWriter;
        if (existing) {
          systemMetadataWriter =
            new DatasetSystemMetadataWriter(metadataStore, datasetInstanceId, props,
                                            dataset, typeMeta.getName(), spec.getDescription());
        } else {
          long createTime = System.currentTimeMillis();
          systemMetadataWriter =
            new DatasetSystemMetadataWriter(metadataStore, datasetInstanceId, props, createTime,
                                            dataset, typeMeta.getName(), spec.getDescription());
        }
        systemMetadataWriter.write();
      } finally {
        if (dataset != null) {
          dataset.close();
        }
      }
    }
  }

  public void drop(Id.DatasetInstance datasetInstanceId, DatasetTypeMeta typeMeta, DatasetSpecification spec)
    throws Exception {
    LOG.info("Dropping dataset with spec: {}, type meta: {}", spec, typeMeta);
    try (DatasetClassLoaderProvider classLoaderProvider =
           new DirectoryClassLoaderProvider(cConf, locationFactory)) {
      DatasetType type = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);

      if (type == null) {
        String msg = String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta);
        LOG.error(msg);
        throw new BadRequestException(msg);
      }

      DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespaceId()), spec);
      admin.drop();
    }

    // Remove metadata for the dataset
    metadataStore.removeMetadata(datasetInstanceId);
  }

  public void truncate(Id.DatasetInstance datasetInstanceId) throws Exception {
    DatasetAdmin datasetAdmin = getDatasetAdmin(datasetInstanceId);
    datasetAdmin.truncate();
  }

  public void upgrade(Id.DatasetInstance datasetInstanceId) throws Exception {
    DatasetAdmin datasetAdmin = getDatasetAdmin(datasetInstanceId);
    datasetAdmin.upgrade();
  }

  private DatasetAdmin getDatasetAdmin(Id.DatasetInstance datasetInstanceId) throws IOException,
    DatasetManagementException, NotFoundException {

    try (SystemDatasetInstantiator datasetInstantiator = datasetInstantiatorFactory.create()) {
      DatasetAdmin admin = datasetInstantiator.getDatasetAdmin(datasetInstanceId);
      if (admin == null) {
        throw new NotFoundException("Couldn't obtain DatasetAdmin for dataset instance " + datasetInstanceId);
      }
      return admin;
    }
  }

  //TODO: CDAP-4627 - Figure out a better way to identify system datasets in user namespaces
  private boolean isUserDataset(Id.DatasetInstance datasetInstanceId) {
    return !Id.Namespace.SYSTEM.equals(datasetInstanceId.getNamespace()) &&
      !"system.queue.config".equals(datasetInstanceId.getId()) &&
      !datasetInstanceId.getId().startsWith("system.sharded.queue") &&
      !datasetInstanceId.getId().startsWith("system.queue") &&
      !datasetInstanceId.getId().startsWith("system.stream");
  }
}