/*
* Copyright 2015 herd contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.finra.herd.rest;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.finra.herd.model.api.xml.BusinessObjectDataStorageFilesCreateRequest;
import org.finra.herd.model.api.xml.BusinessObjectDataStorageFilesCreateResponse;
import org.finra.herd.model.api.xml.SchemaColumn;
import org.finra.herd.model.api.xml.StorageFile;
import org.finra.herd.model.dto.S3FileTransferRequestParamsDto;
import org.finra.herd.model.jpa.BusinessObjectDataEntity;
import org.finra.herd.model.jpa.BusinessObjectDataStatusEntity;
import org.finra.herd.model.jpa.BusinessObjectDefinitionEntity;
import org.finra.herd.model.jpa.BusinessObjectFormatEntity;
import org.finra.herd.model.jpa.DataProviderEntity;
import org.finra.herd.model.jpa.FileTypeEntity;
import org.finra.herd.model.jpa.NamespaceEntity;
import org.finra.herd.model.jpa.StorageEntity;
import org.finra.herd.model.jpa.StorageUnitEntity;
import org.finra.herd.model.jpa.StorageUnitStatusEntity;
/**
* This class tests CreateBusinessObjectDataStorageFiles functionality within the business object data storage file REST controller.
*/
public class BusinessObjectDataStorageFileRestControllerTest extends AbstractRestTest
{
private static final String FILE_PATH_1 = "file1";
private static final String FILE_PATH_2 = "file2";
private static final String PARTITION_KEY_2 = "pk2_" + Math.random();
private static final String PARTITION_KEY_3 = "pk3_" + Math.random();
private static final String PARTITION_KEY_4 = "pk4_" + Math.random();
private static final String PARTITION_KEY_5 = "pk5_" + Math.random();
private static final String PARTITION_VALUE_2 = "pv2_" + Math.random();
private static final String PARTITION_VALUE_3 = "pv3_" + Math.random();
private static final String PARTITION_VALUE_4 = "pv4_" + Math.random();
private static final String PARTITION_VALUE_5 = "pv5_" + Math.random();
private static final List<String> SUB_PARTITION_VALUES = Arrays.asList(PARTITION_VALUE_2, PARTITION_VALUE_3, PARTITION_VALUE_4, PARTITION_VALUE_5);
private static final String testS3KeyPrefix =
getExpectedS3KeyPrefix(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY,
PARTITION_VALUE, null, null, DATA_VERSION);
private Path localTempPath;
/**
* Sets up the test environment.
*/
@Before
public void setupEnv() throws IOException
{
// Create a local temp directory.
localTempPath = Files.createTempDirectory(null);
}
/**
* Cleans up the local temp directory and S3 test path that we are using.
*/
@After
public void cleanEnv() throws IOException
{
// Clean up the local directory.
FileUtils.deleteDirectory(localTempPath.toFile());
// Clean up the destination S3 folder.
S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
s3FileTransferRequestParamsDto.setS3KeyPrefix(testS3KeyPrefix);
s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
}
@Test
public void testCreateBusinessObjectDataStorageFiles()
{
createDataWithSubPartitions();
BusinessObjectDataStorageFilesCreateRequest request =
new BusinessObjectDataStorageFilesCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE,
SUB_PARTITION_VALUES, DATA_VERSION, STORAGE_NAME, Arrays.asList(createFile(FILE_PATH_2, FILE_SIZE_1_KB, ROW_COUNT_1000)),
NO_DISCOVER_STORAGE_FILES);
BusinessObjectDataStorageFilesCreateResponse response = businessObjectDataStorageFileRestController.createBusinessObjectDataStorageFiles(request);
// Validate the returned object.
businessObjectDataServiceTestHelper
.validateBusinessObjectDataStorageFilesCreateResponse(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION,
PARTITION_VALUE, SUB_PARTITION_VALUES, DATA_VERSION, STORAGE_NAME, request.getStorageFiles(), response);
}
private StorageFile createFile(String filePath, Long size, Long rowCount)
{
StorageFile f = new StorageFile();
f.setFilePath(filePath);
f.setFileSizeBytes(size);
f.setRowCount(rowCount);
return f;
}
private void createDataWithSubPartitions()
{
NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
DataProviderEntity dataProviderEntity = dataProviderDaoTestHelper.createDataProviderEntity(DATA_PROVIDER_NAME);
BusinessObjectDefinitionEntity businessObjectDefinitionEntity = businessObjectDefinitionDaoTestHelper
.createBusinessObjectDefinitionEntity(namespaceEntity, BDEF_NAME, dataProviderEntity, NO_BDEF_DESCRIPTION, NO_BDEF_DISPLAY_NAME, NO_ATTRIBUTES,
NO_SAMPLE_DATA_FILES);
FileTypeEntity fileTypeEntity = fileTypeDaoTestHelper.createFileTypeEntity(FORMAT_FILE_TYPE_CODE, FORMAT_DESCRIPTION);
List<SchemaColumn> schemaColumns = new ArrayList<>();
{
SchemaColumn schemaColumn = new SchemaColumn();
schemaColumn.setName(PARTITION_KEY);
schemaColumn.setType("STRING");
schemaColumns.add(schemaColumn);
}
{
SchemaColumn schemaColumn = new SchemaColumn();
schemaColumn.setName(PARTITION_KEY_2);
schemaColumn.setType("STRING");
schemaColumns.add(schemaColumn);
}
{
SchemaColumn schemaColumn = new SchemaColumn();
schemaColumn.setName(PARTITION_KEY_3);
schemaColumn.setType("STRING");
schemaColumns.add(schemaColumn);
}
{
SchemaColumn schemaColumn = new SchemaColumn();
schemaColumn.setName(PARTITION_KEY_4);
schemaColumn.setType("STRING");
schemaColumns.add(schemaColumn);
}
{
SchemaColumn schemaColumn = new SchemaColumn();
schemaColumn.setName(PARTITION_KEY_5);
schemaColumn.setType("STRING");
schemaColumns.add(schemaColumn);
}
BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper
.createBusinessObjectFormatEntity(businessObjectDefinitionEntity, FORMAT_USAGE_CODE, fileTypeEntity, FORMAT_VERSION, null, true, PARTITION_KEY,
null, NO_ATTRIBUTES, null, null, null, schemaColumns, null);
BusinessObjectDataStatusEntity businessObjectDataStatusEntity =
businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS, DESCRIPTION, BDATA_STATUS_PRE_REGISTRATION_FLAG_SET);
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper
.createBusinessObjectDataEntity(businessObjectFormatEntity, PARTITION_VALUE, SUB_PARTITION_VALUES, DATA_VERSION, true,
businessObjectDataStatusEntity.getCode());
StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME);
StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper
.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, FILE_PATH_1, FILE_SIZE_1_KB, null);
}
}