/* * Copyright 2016 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package com.emc.storageos.api.service.impl.resource.utils; import com.emc.storageos.api.service.impl.resource.*; import com.emc.storageos.cinder.CinderConstants; import com.emc.storageos.db.client.URIUtil; import com.emc.storageos.db.client.model.*; import com.emc.storageos.keystone.restapi.model.response.KeystoneTenant; import com.emc.storageos.keystone.restapi.utils.KeystoneUtils; import com.emc.storageos.model.project.ProjectElement; import com.emc.storageos.model.project.ProjectParam; import com.emc.storageos.model.tenant.TenantOrgRestRep; import com.emc.storageos.security.authentication.InternalTenantServiceClient; import com.emc.storageos.svcs.errorhandling.resources.APIException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.URI; import java.util.*; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; public class OpenStackSynchronizationTask extends ResourceService { private static final Logger _log = LoggerFactory.getLogger(OpenStackSynchronizationTask.class); // Constants // Interval delay between each execution in seconds. public static final int DEFAULT_INTERVAL_DELAY = 900; // Maximum time in seconds for a timeout when awaiting for termination. private static final int MAX_TERMINATION_TIME = 120; // Minimum interval in seconds. public static final int MIN_INTERVAL_DELAY = 10; // The number of threads to keep in the pool public static final int NUMBER_OF_THREADS = 1; private static final String OPENSTACK = "OpenStack"; // Services private KeystoneUtils _keystoneUtilsService; private InternalTenantServiceClient _internalTenantServiceClient; private ScheduledExecutorService _dataCollectionExecutorService; private ScheduledFuture _synchronizationTask; public void setInternalTenantServiceClient(InternalTenantServiceClient internalTenantServiceClient) { this._internalTenantServiceClient = internalTenantServiceClient; } public ScheduledFuture getSynchronizationTask() { return _synchronizationTask; } public void setKeystoneUtilsService(KeystoneUtils keystoneUtilsService) { this._keystoneUtilsService = keystoneUtilsService; } public void start(int interval) throws Exception { _log.info("Start OpenStack Synchronization Task"); _dataCollectionExecutorService = Executors.newScheduledThreadPool(NUMBER_OF_THREADS); // Schedule task at fixed interval. _synchronizationTask = _dataCollectionExecutorService.scheduleAtFixedRate( new SynchronizationScheduler(), interval, interval, TimeUnit.SECONDS); } public void stop() { _log.info("Stop OpenStack Synchronization Task"); try { _dataCollectionExecutorService.shutdown(); _dataCollectionExecutorService.awaitTermination(MAX_TERMINATION_TIME, TimeUnit.SECONDS); _synchronizationTask = null; } catch (Exception e) { _log.error("TimeOut occurred after waiting Client Threads to finish"); } } /** * Reschedule synchronization task with given interval. Previous task is canceled and a new one is scheduled. * * @param newInterval New interval delay between each execution in seconds. */ public void rescheduleTask(int newInterval) { if (_synchronizationTask != null && newInterval >= MIN_INTERVAL_DELAY) { _synchronizationTask.cancel(false); _synchronizationTask = _dataCollectionExecutorService .scheduleAtFixedRate(new SynchronizationScheduler(), newInterval, newInterval, TimeUnit.SECONDS); _log.debug("Synchronization task has been rescheduled with {}s interval.", newInterval); } else { throw APIException.internalServerErrors.rescheduleSynchronizationTaskError(); } } /** * Retrieves Keystone Authentication Provider from CoprHD database. * * @return Keystone Authentication Provider. */ public AuthnProvider getKeystoneProvider() { return _keystoneUtilsService.getKeystoneProvider(); } /** * Retrieves interval value of Sync Task from Keystone Authentication Provider. * * @param keystoneProvider Keystone Authentication Provider. * * @return OpenStack Synchronization Task interval. */ public int getTaskInterval(AuthnProvider keystoneProvider) { if (keystoneProvider != null && keystoneProvider.getTenantsSynchronizationOptions() != null) { String taskInterval = getIntervalFromTenantSyncSet(keystoneProvider.getTenantsSynchronizationOptions()); if (taskInterval != null) { return Integer.parseInt(taskInterval); } } throw APIException.internalServerErrors.targetIsNullOrEmpty("keystone provider or tenantsSynchronizationOptions"); } /** * Retrieves interval value from Tenants Synchronization Options in Keystone Authentication Provider. * * @param tenantsSynchronizationOptions Tenants Synchronization Options. * * @return interval. */ public String getIntervalFromTenantSyncSet(StringSet tenantsSynchronizationOptions) { for (String option : tenantsSynchronizationOptions) { // There is only ADDITION, DELETION and interval in this StringSet. if (!AuthnProvider.TenantsSynchronizationOptions.ADDITION.toString().equals(option) && !AuthnProvider.TenantsSynchronizationOptions.DELETION.toString().equals(option)) { return option; } } return null; } /** * Finds CoprHD Tenants with OpenStack ID that are different from their reflection in OpenStack. * Those found Tenants need to be updated. * * @param osTenantList List of OpenStack Tenants. * @param coprhdTenantList List of CoprHD Tenants related to OpenStack. * @return List of CoprHD Tenants that needs to be updated. */ private List<TenantOrg> getListOfTenantsToUpdate(List<KeystoneTenant> osTenantList, List<TenantOrg> coprhdTenantList) { List<TenantOrg> tenantsToUpdate = null; ListIterator<KeystoneTenant> osIter = osTenantList.listIterator(); while (osIter.hasNext()) { ListIterator<TenantOrg> coprhdIter = coprhdTenantList.listIterator(); KeystoneTenant osTenant = osIter.next(); // Update information about this Tenant in CoprHD database. createOrUpdateOpenstackTenantInCoprhd(osTenant); while (coprhdIter.hasNext()) { TenantOrg coprhdTenant = coprhdIter.next(); String tenantMapping = _keystoneUtilsService.getCoprhdTenantUserMapping(coprhdTenant); String tenantId = _keystoneUtilsService.getTenantIdFromUserMapping(tenantMapping); if (tenantId.equals(osTenant.getId())) { if (!areTenantsIdentical(osTenant, coprhdTenant)) { coprhdTenant.setDescription(osTenant.getDescription()); String tenantName = OPENSTACK + " " + osTenant.getName(); coprhdTenant.setLabel(tenantName); if (tenantsToUpdate == null) { tenantsToUpdate = new ArrayList<>(); } tenantsToUpdate.add(coprhdTenant); } coprhdIter.remove(); osIter.remove(); } } } return tenantsToUpdate; } /** * Updates or creates CoprHD representation of OpenStack Tenant. * * @param tenant OpenStack Tenant. * @return Updated or Created Tenant. */ private OSTenant createOrUpdateOpenstackTenantInCoprhd(KeystoneTenant tenant) { OSTenant osTenant = _keystoneUtilsService.findOpenstackTenantInCoprhd(tenant.getId()); if (osTenant != null) { if (!osTenant.getDescription().equals(tenant.getDescription())) { osTenant.setDescription(tenant.getDescription()); } if (!osTenant.getName().equals(tenant.getName())) { osTenant.setName(tenant.getName()); } if (osTenant.getEnabled() != Boolean.parseBoolean(tenant.getEnabled())) { osTenant.setEnabled(Boolean.parseBoolean(tenant.getEnabled())); } _dbClient.updateObject(osTenant); } else { osTenant = _keystoneUtilsService.mapToOsTenant(tenant); osTenant.setId(URIUtil.createId(OSTenant.class)); _dbClient.createObject(osTenant); } return osTenant; } /** * Compares OpenStack Tenant with CoprHD Tenant (both need to have same OpenStack ID). * * @param osTenant OpenStack Tenant. * @param coprhdTenant CoprHD Tenant related to OpenStack. * @return True if tenants are identical, false otherwise. */ private boolean areTenantsIdentical(KeystoneTenant osTenant, TenantOrg coprhdTenant) { String osTenantName = OPENSTACK + " " + osTenant.getName(); if (!osTenantName.equals(coprhdTenant.getLabel())) { return false; } if (!osTenant.getDescription().equals(coprhdTenant.getDescription())) { return false; } return true; } /** * Creates a CoprHD Tenant for given OpenStack Tenant. * Sends internal POST API call to InternalTenantsService in order to create Tenant. * * @param tenant OpenStack Tenant. * * @return URI of newly created Tenant. */ public URI createTenant(KeystoneTenant tenant) { TenantOrgRestRep tenantResp = _internalTenantServiceClient.createTenant(_keystoneUtilsService.prepareTenantParam(tenant)); return tenantResp.getId(); } /** * Creates a CoprHD Project for given Tenant. * Sends internal POST API call to InternalTenantsService in order to create Project. * * @param tenantOrgId ID of the Project owner. * @param tenant OpenStack Tenant. * * @return URI of newly created Project. */ public URI createProject(URI tenantOrgId, KeystoneTenant tenant) { ProjectParam projectParam = new ProjectParam(tenant.getName() + CinderConstants.PROJECT_NAME_SUFFIX); ProjectElement projectResp = _internalTenantServiceClient.createProject(tenantOrgId, projectParam); return projectResp.getId(); } /** * Starts synchronization between CoprHD and OpenStack Tenants (i.e. starts Synchronization Task). * * @param interval Task interval. */ public void startSynchronizationTask(int interval) { try { if (interval > MIN_INTERVAL_DELAY) { start(interval); } } catch (Exception e) { _log.error("Exception when trying to start synchronization task: {}", e.getMessage()); } } /** * Stops synchronization between CoprHD and OpenStack Tenants. * */ public void stopSynchronizationTask() { try { stop(); } catch (Exception e) { _log.error("Exception when trying to stop synchronization task: {}", e.getMessage()); } } private class SynchronizationScheduler implements Runnable { @Override public void run() { try { // List of OpenStack Tenants. List<KeystoneTenant> osTenantList = _keystoneUtilsService.getOpenStackTenants(); List<KeystoneTenant> openstackTenants = new ArrayList<>(osTenantList); // List of CoprHD Tenants. List<TenantOrg> coprhdTenantList = _keystoneUtilsService.getCoprhdTenantsWithOpenStackId(); // List of CoprHD Tenants to update. List<TenantOrg> tenantsToUpdate = getListOfTenantsToUpdate(osTenantList, coprhdTenantList); int size = (tenantsToUpdate == null) ? 0 : tenantsToUpdate.size(); _log.debug("Tenants to update: {}, to create: {}, to delete: {}", size, osTenantList.size(), coprhdTenantList.size()); AuthnProvider keystoneProvider = getKeystoneProvider(); // Update every Tenant on tenantsToUpdate list. if (tenantsToUpdate != null && !tenantsToUpdate.isEmpty()) { for (TenantOrg tenant : tenantsToUpdate) { _dbClient.updateObject(tenant); } } _internalTenantServiceClient.setServer(_keystoneUtilsService.getVIP()); StringSet syncOptions = keystoneProvider.getTenantsSynchronizationOptions(); // Check whether Automatic Addition is enabled. if (!osTenantList.isEmpty() && syncOptions.contains(AuthnProvider.TenantsSynchronizationOptions.ADDITION.toString())) { for (KeystoneTenant tenant : osTenantList) { OSTenant osTenant = _keystoneUtilsService.findOpenstackTenantInCoprhd(tenant.getId()); if (osTenant != null && !osTenant.getExcluded()) { URI tenantOrgId = createTenant(tenant); URI projectId = createProject(tenantOrgId, tenant); _keystoneUtilsService.tagProjectWithOpenstackId(projectId, tenant.getId(), tenantOrgId.toString()); } } } // Synchronize OSTenants with Tenants in OpenStack. List<URI> osTenantURI = _dbClient.queryByType(OSTenant.class, true); Iterator<OSTenant> osTenantIter = _dbClient.queryIterativeObjects(OSTenant.class, osTenantURI); while (osTenantIter.hasNext()) { OSTenant osTenant = osTenantIter.next(); // Maps openstack Tenants to a map with IDs only and then filters for specific ID. int matches = (int) openstackTenants.stream().map(KeystoneTenant::getId).filter(id -> id.equals(osTenant.getOsId())).count(); // Remove OSTenant when Tenant with the same ID in OpenStack is deleted. if (matches < 1) { _dbClient.removeObject(osTenant); } } // Removes CoprHD Tenants related to OpenStack that are absent in OS. if (!coprhdTenantList.isEmpty() && syncOptions.contains(AuthnProvider.TenantsSynchronizationOptions.DELETION.toString())) { List<URI> projectURI = _dbClient.queryByType(Project.class, true); Iterator<Project> projectIter = _dbClient.queryIterativeObjects(Project.class, projectURI); for (TenantOrg tenant : coprhdTenantList) { while (projectIter.hasNext()) { Project project = projectIter.next(); if (project.getTenantOrg().getURI().equals(tenant.getId())) { ArgValidator.checkReference(Project.class, project.getId(), checkForDelete(project)); _dbClient.markForDeletion(project); } } ArgValidator.checkReference(TenantOrg.class, tenant.getId(), checkForDelete(tenant)); _dbClient.markForDeletion(tenant); } } } catch (Exception e) { _log.error("Exception caught when trying to run OpenStack Synchronization job: {}", e); } } } }