/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.usergrid.management.importer;
import com.amazonaws.SDKGlobalConfiguration;
import com.google.common.collect.BiMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.Service;
import com.google.inject.Module;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.batch.service.JobSchedulerService;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.management.export.ExportService;
import org.apache.usergrid.management.export.S3Export;
import org.apache.usergrid.management.export.S3ExportImpl;
import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.entities.Import;
import org.apache.usergrid.persistence.entities.JobData;
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.services.notifications.QueueListener;
import org.apache.usergrid.setup.ConcurrentProcessSingleton;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.netty.config.NettyPayloadModule;
import org.junit.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.UUID;
import static org.hamcrest.core.Is.is;
import static org.hamcrest.core.IsNot.not;
import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class ImportServiceIT {
private static final Logger logger = LoggerFactory.getLogger(ImportServiceIT.class);
// app-level data generated only once
private static UserInfo adminUser;
private static OrganizationInfo organization;
private static UUID applicationId;
QueueListener listener;
final String bucketName = System.getProperty( "bucketName" )
+ RandomStringUtils.randomAlphanumeric(10).toLowerCase();
@Rule
public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
@ClassRule
public static final ServiceITSetup setup =
new ServiceITSetupImpl( );
@BeforeClass
public static void setup() throws Exception {
String username = "test"+ UUIDUtils.newTimeUUID();
// start the scheduler after we're all set up
// start the scheduler after we're all set up
JobSchedulerService jobScheduler = ConcurrentProcessSingleton
.getInstance().getSpringResource().getBean( JobSchedulerService.class );
if ( jobScheduler.state() != Service.State.RUNNING ) {
jobScheduler.startAsync();
jobScheduler.awaitRunning();
}
//creates sample test application
adminUser = setup.getMgmtSvc().createAdminUser( null,
username, username, username+"@test.com", username, false, false );
organization = setup.getMgmtSvc().createOrganization( username, adminUser, true );
applicationId = setup.getMgmtSvc().createApplication( organization.getUuid(), username+"app" ).getId();
}
@Before
public void before() {
boolean configured =
!StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR))
&& !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR))
&& !StringUtils.isEmpty(System.getProperty("bucketName"));
if ( !configured ) {
logger.warn("Skipping test because {}, {} and bucketName not " +
"specified as system properties, e.g. in your Maven settings.xml file.",
new Object[] {
SDKGlobalConfiguration.SECRET_KEY_ENV_VAR,
SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR
});
}
Assume.assumeTrue( configured );
}
@After
public void after() throws Exception {
if(listener != null) {
listener.stop();
listener = null;
}
}
// test case to check if application is imported correctly
@Test
@Ignore("Pending merge of export-feature branch. Import organization not supported")
public void testImportApplication() throws Exception {
EntityManager em = setup.getEmf().getEntityManager( applicationId );
// Create five user entities (we already have one admin user)
List<Entity> entities = new ArrayList<>();
for ( int i = 0; i < 5; i++ ) {
Map<String, Object> userProperties = new LinkedHashMap<>();
userProperties.put( "parameter1", "user" + i );
userProperties.put( "parameter2", "user" + i + "@test.com" );
entities.add( em.create( "custom", userProperties ) );
}
// Creates connections
em.createConnection( new SimpleEntityRef( "custom", entities.get(0).getUuid() ),
"related", new SimpleEntityRef( "custom", entities.get(1).getUuid() ) );
em.createConnection( new SimpleEntityRef( "custom", entities.get(1).getUuid() ),
"related", new SimpleEntityRef( "custom", entities.get(0).getUuid() ) );
logger.debug("\n\nExport the application\n\n");
// Export the application which needs to be tested for import
ExportService exportService = setup.getExportService();
S3Export s3Export = new S3ExportImpl();
HashMap<String, Object> payload = payloadBuilder();
payload.put( "organizationId", organization.getUuid());
payload.put( "applicationId", applicationId );
// Schedule the export job
UUID exportUUID = exportService.schedule( payload );
// Create and initialize jobData returned in JobExecution.
JobData jobData = jobExportDataCreator(payload, exportUUID, s3Export);
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
// Export the application and wait for the export job to finish
exportService.doExport( jobExecution );
while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {
// wait...
}
logger.debug("Import the application\n\n");
// import
S3Import s3Import = new S3ImportImpl();
ImportService importService = setup.getImportService();
// scheduele the import job
final Import importEntity = importService.schedule( null, payload );
final UUID importUUID = importEntity.getUuid();
//create and initialize jobData returned in JobExecution.
jobData = jobImportDataCreator( payload,importUUID, s3Import );
jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
// import the application file and wait for it to finish
importService.doImport(jobExecution);
while ( !importService.getState( importUUID ).equals( "FINISHED" ) ) {
// wait...
}
if (logger.isDebugEnabled()) {
logger.debug("Verify Import");
}
try {
//checks if temp import files are created i.e. downloaded from S3
//assertThat(importService.getEphemeralFile().size(), is(not(0)));
Set<String> collections = em.getApplicationCollections();
// check if all collections in the application are updated
for (String collectionName : collections) {
if (logger.isDebugEnabled()) {
logger.debug("Checking collection {}", collectionName);
}
Results collection = em.getCollection(applicationId, collectionName, null, Level.ALL_PROPERTIES);
for (Entity eachEntity : collection.getEntities() ) {
if (logger.isDebugEnabled()) {
logger.debug("Checking entity {} {}:{}",
new Object[]{eachEntity.getName(), eachEntity.getType(), eachEntity.getUuid()});
}
//check for dictionaries --> checking permissions in the dictionaries
EntityRef er;
Map<Object, Object> dictionaries;
//checking for permissions for the roles collection
if (collectionName.equals("roles")) {
if (eachEntity.getName().equalsIgnoreCase("admin")) {
er = eachEntity;
dictionaries = em.getDictionaryAsMap(er, "permissions");
assertThat(dictionaries.size(), is(not(0))); // admin has permission
} else {
er = eachEntity;
dictionaries = em.getDictionaryAsMap(er, "permissions");
assertThat(dictionaries.size(), is(0)); // other roles do not
}
}
}
if (collectionName.equals("customs")) {
// check if connections are created for only the 1st 2 entities in the custom collection
Results r;
List<ConnectionRef> connections;
for (int i = 0; i < 2; i++) {
r = em.getTargetEntities(entities.get(i), "related", null, Level.IDS);
connections = r.getConnections();
assertNotNull(connections);
}
}
}
}
finally {
//delete bucket
deleteBucket();
}
}
// test case to check if all applications file for an organization are imported correctly
@Test
@Ignore("Pending merge of export-feature branch. Import organization not supported")
public void testImportOrganization() throws Exception {
// creates 5 entities in usertests collection
EntityManager em = setup.getEmf().getEntityManager( applicationId );
//intialize user object to be posted
Map<String, Object> userProperties = null;
Entity entity[] = new Entity[5];
//creates entities
for ( int i = 0; i < 5; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "name", "user" + i );
userProperties.put( "email", "user" + i + "@test.com" );
entity[i] = em.create( "usertests", userProperties );
em.getCollections(entity[i]).contains("usertests");
}
//creates test connections between first 2 entities in usertests collection
ConnectedEntityRef ref = em.createConnection( entity[0], "related", entity[1]);
em.createConnection( entity[1], "related", entity[0]);
//create 2nd test application, add entities to it, create connections and set permissions
createAndSetup2ndApplication();
//export all applications in an organization
ExportService exportService = setup.getExportService();
S3Export s3Export = new S3ExportImpl();
HashMap<String, Object> payload = payloadBuilder();
payload.put( "organizationId", organization.getUuid());
//schdeule the export job
UUID exportUUID = exportService.schedule( payload );
//create and initialize jobData returned in JobExecution.
JobData jobData = jobExportDataCreator(payload, exportUUID, s3Export);
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
//export organization data and wait for the export job to finish
exportService.doExport( jobExecution );
while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {
;
}
//TODO: can check if the temp files got created
// import
S3Import s3Import = new S3ImportImpl();
ImportService importService = setup.getImportService();
//schedule the import job
final Import importEntity = importService.schedule( null, payload );
final UUID importUUID = importEntity.getUuid();
//create and initialize jobData returned in JobExecution.
jobData = jobImportDataCreator( payload,importUUID, s3Import );
jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
//import the all application files for the organization and wait for the import to finish
importService.doImport(jobExecution);
while ( !importService.getState( importUUID ).equals( Import.State.FINISHED ) ) {
;
}
try {
//checks if temp import files are created i.e. downloaded from S3
//assertThat(importService.getEphemeralFile().size(), is(not(0)));
//get all applications for an organization
BiMap<UUID, String> applications =
setup.getMgmtSvc().getApplicationsForOrganization(organization.getUuid());
for (BiMap.Entry<UUID, String> app : applications.entrySet()) {
//check if all collections-entities are updated - created and modified should be different
UUID appID = app.getKey();
em = setup.getEmf().getEntityManager(appID);
Set<String> collections = em.getApplicationCollections();
Iterator<String> itr = collections.iterator();
while (itr.hasNext()) {
String collectionName = itr.next();
Results collection = em.getCollection(appID, collectionName, null, Level.ALL_PROPERTIES);
List<Entity> entities = collection.getEntities();
if (collectionName.equals("usertests")) {
// check if connections are created for only the 1st 2 entities in user collection
Results r;
List<ConnectionRef> connections;
for (int i = 0; i < 2; i++) {
r = em.getTargetEntities(entities.get(i), "related", null, Level.IDS);
connections = r.getConnections();
assertNotNull(connections);
}
//check if dictionary is created
EntityRef er;
Map<Object, Object> dictionaries1, dictionaries2;
for (int i = 0; i < 3; i++) {
er = entities.get(i);
dictionaries1 = em.getDictionaryAsMap(er, "connected_types");
dictionaries2 = em.getDictionaryAsMap(er, "connecting_types");
if (i == 2) {
//for entity 2, these should be empty
assertThat(dictionaries1.size(), is(0));
assertThat(dictionaries2.size(), is(0));
} else {
assertThat(dictionaries1.size(), is(not(0)));
assertThat(dictionaries2.size(), is(not(0)));
}
}
}
}
}
}
finally {
//delete bucket
deleteBucket();
}
}
/**
* Test to schedule a job with null config
*/
@Test(expected=NullPointerException.class)
public void testScheduleJobWithNullConfig() throws Exception {
HashMap<String, Object> payload = null;
ImportService importService = setup.getImportService();
final Import importEntity = importService.schedule( null, payload );
assertNull(importEntity);
}
/**
* Test to get state of a job with null UUID
*/
@Test(expected = NullPointerException.class)
public void testGetStateWithNullUUID() throws Exception {
UUID uuid= null;
ImportService importService = setup.getImportService();
Import.State state = importService.getState( uuid );
}
/**
* Test to get state of a job with fake UUID
*/
@Test(expected = EntityNotFoundException.class)
public void testGetStateWithFakeUUID() throws Exception {
UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
ImportService importService = setup.getImportService();
Import.State state = importService.getState( fake );
}
/**
* Test to get error message of a job with null state
*/
@Test
public void testErrorMessageWithNullState() throws Exception {
UUID state = null;
ImportService importService = setup.getImportService();
String error = importService.getErrorMessage(state);
assertEquals(error,"UUID passed in cannot be null");
}
/**
* Test to get error message of a job with fake UUID
*/
@Test
public void testErrorMessageWithFakeUUID() throws Exception {
UUID state = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
ImportService importService = setup.getImportService();
String error = importService.getErrorMessage(state);
assertEquals(error,"No Such Element found");
}
/**
* Test to the doImport method with null organziation ID
*/
@Test
@Ignore("Pending merge of export-feature branch. Import organization not supported")
public void testDoImportWithNullOrganizationID() throws Exception {
// import
S3Import s3Import = new S3ImportImpl();
ImportService importService = setup.getImportService();
HashMap<String, Object> payload = payloadBuilder();
//schedule the import job
final Import importEntity = importService.schedule( null, payload );
final UUID importUUID = importEntity.getUuid();
//create and initialize jobData returned in JobExecution.
JobData jobData = jobImportDataCreator(payload, importUUID, s3Import);
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
importService.doImport(jobExecution);
assertEquals(importService.getState(importUUID),Import.State.FAILED);
}
/**
* Test to the doImport method with fake organization ID
*/
@Test
@Ignore("Pending merge of export-feature branch. Import organization not supported")
public void testDoImportWithFakeOrganizationID() throws Exception {
UUID fakeOrgId = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
// import
S3Import s3Import = new S3ImportImpl();
ImportService importService = setup.getImportService();
HashMap<String, Object> payload = payloadBuilder();
payload.put("organizationId",fakeOrgId);
//schedule the import job
final Import importEntity = importService.schedule( null, payload );
final UUID importUUID = importEntity.getUuid();
//create and initialize jobData returned in JobExecution.
JobData jobData = jobImportDataCreator(payload, importUUID, s3Import);
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
//import the all application files for the organization and wait for the import to finish
importService.doImport(jobExecution);
assertEquals(Import.State.FAILED, importService.getState(importUUID));
}
/**
* Test to the doImport method with fake application ID
*/
@Test
@Ignore("Pending merge of export-feature branch. Import application not supported")
public void testDoImportWithFakeApplicationID() throws Exception {
UUID fakeappId = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
// import
S3Import s3Import = new S3ImportImpl();
ImportService importService = setup.getImportService();
HashMap<String, Object> payload = payloadBuilder();
payload.put("organizationId",organization.getUuid());
payload.put("applicationId",fakeappId);
//schedule the import job
final Import importEntity = importService.schedule( null, payload );
final UUID importUUID = importEntity.getUuid();
//create and initialize jobData returned in JobExecution.
JobData jobData = jobImportDataCreator(payload, importUUID, s3Import);
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
//import the application files for the organization and wait for the import to finish
importService.doImport(jobExecution);
assertEquals(Import.State.FAILED, importService.getState(importUUID));
}
/**
* Test to the doImport Collection method with fake application ID
*/
@Test
@Ignore("Pending merge of export-feature branch. Import application not supported")
public void testDoImportCollectionWithFakeApplicationID() throws Exception {
UUID fakeappId = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
// import
S3Import s3Import = new S3ImportImpl();
ImportService importService = setup.getImportService();
HashMap<String, Object> payload = payloadBuilder();
payload.put("organizationId",organization.getUuid());
payload.put("applicationId",fakeappId);
payload.put("collectionName","custom-test");
//schedule the import job
final Import importEntity = importService.schedule( null, payload );
final UUID importUUID = importEntity.getUuid();
//create and initialize jobData returned in JobExecution.
JobData jobData = jobImportDataCreator(payload, importUUID, s3Import);
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
//import the all collection files for the organization-application and wait for the import to finish
importService.doImport(jobExecution);
assertEquals(importService.getState(importUUID),Import.State.FAILED);
}
/*Creates fake payload for testing purposes.*/
public HashMap<String, Object> payloadBuilder() {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> properties = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
storage_info.put( "bucket_location", bucketName );
storage_info.put( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR,
System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
storage_info.put( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR,
System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ) );
properties.put( "storage_provider", "s3" );
properties.put( "storage_info", storage_info );
payload.put( "path","test-organization/test-app" );
payload.put( "properties", properties );
return payload;
}
//creates fake import job
public JobData jobImportDataCreator(HashMap<String, Object> payload,UUID importUUID,S3Import s3Import) {
JobData jobData = new JobData();
jobData.setProperty( "jobName", "importJob" );
jobData.setProperty( "importInfo", payload );
jobData.setProperty( "importId", importUUID );
jobData.setProperty( "s3Import", s3Import );
return jobData;
}
//creates fake export job
public JobData jobExportDataCreator(HashMap<String, Object> payload,UUID exportUUID,S3Export s3Export) {
JobData jobData = new JobData();
jobData.setProperty( "jobName", "exportJob" );
jobData.setProperty( "exportInfo", payload );
jobData.setProperty( "exportId", exportUUID );
jobData.setProperty( "s3Export", s3Export);
return jobData;
}
// delete the s3 bucket which was created for testing
public void deleteBucket() {
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
Properties overrides = new Properties();
overrides.setProperty( "s3" + ".identity", accessId );
overrides.setProperty( "s3" + ".credential", secretKey );
Blob bo = null;
BlobStore blobStore = null;
final Iterable<? extends Module> MODULES = ImmutableSet
.of(new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
new NettyPayloadModule());
BlobStoreContext context =
ContextBuilder.newBuilder("s3").credentials( accessId, secretKey ).modules( MODULES )
.overrides( overrides ).buildView( BlobStoreContext.class );
blobStore = context.getBlobStore();
blobStore.deleteContainer( bucketName );
}
//creates 2nd application for testing import from an organization having multiple applications
void createAndSetup2ndApplication() throws Exception {
UUID appId = setup.getMgmtSvc().createApplication( organization.getUuid(), "test-app-2" ).getId();
EntityManager emTest = setup.getEmf().getEntityManager(appId);
Map<String, Object> userProperties = null;
Entity entityTest[] = new Entity[5];
//creates entities and set permissions
for ( int i = 0; i < 5; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "name", "user" + i );
userProperties.put( "email", "user" + i + "@test.com" );
entityTest[i] = emTest.create( "testobject", userProperties );
}
//create connection
emTest.createConnection( new SimpleEntityRef( "testobject", entityTest[0].getUuid()),
"related",
new SimpleEntityRef( "testobject", entityTest[1].getUuid()));
emTest.createConnection( new SimpleEntityRef( "testobject", entityTest[1].getUuid()),
"related",
new SimpleEntityRef( "testobject", entityTest[0].getUuid()));
}
}