PatchMigration_J10002.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.ranger.patch;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.ranger.biz.RangerBizUtil;
import org.apache.ranger.biz.ServiceDBStore;
import org.apache.ranger.common.AppConstants;
import org.apache.ranger.common.JSONUtil;
import org.apache.ranger.common.RangerCommonEnums;
import org.apache.ranger.common.SearchCriteria;
import org.apache.ranger.common.ServiceUtil;
import org.apache.ranger.common.StringUtil;
import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.entity.XXAsset;
import org.apache.ranger.entity.XXAuditMap;
import org.apache.ranger.entity.XXGroup;
import org.apache.ranger.entity.XXPolicy;
import org.apache.ranger.entity.XXPolicyConditionDef;
import org.apache.ranger.entity.XXPortalUser;
import org.apache.ranger.entity.XXResource;
import org.apache.ranger.entity.XXServiceConfigDef;
import org.apache.ranger.entity.XXServiceDef;
import org.apache.ranger.entity.XXUser;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerService;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
import org.apache.ranger.service.RangerPolicyService;
import org.apache.ranger.service.XPermMapService;
import org.apache.ranger.service.XPolicyService;
import org.apache.ranger.util.CLIUtil;
import org.apache.ranger.view.VXPermMap;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

@Component
public class PatchMigration_J10002 extends BaseLoader {
	private static final Logger logger = Logger.getLogger(PatchMigration_J10002.class);

	@Autowired
	RangerDaoManager daoMgr;

	@Autowired
	ServiceDBStore svcDBStore;

	@Autowired
	JSONUtil jsonUtil;

	@Autowired
	RangerPolicyService policyService;

	@Autowired
	StringUtil stringUtil;

	@Autowired
	XPolicyService xPolService;

	@Autowired
	XPermMapService xPermMapService;
	
	@Autowired
	RangerBizUtil bizUtil;

	private static int policyCounter = 0;
	private static int serviceCounter = 0;

	static Set<String> unsupportedLegacyPermTypes = new HashSet<String>();

	static {
		unsupportedLegacyPermTypes.add("Unknown");
		unsupportedLegacyPermTypes.add("Reset");
		unsupportedLegacyPermTypes.add("Obfuscate");
		unsupportedLegacyPermTypes.add("Mask");
	}

	public static void main(String[] args) {
		logger.info("main()");
		try {
			PatchMigration_J10002 loader = (PatchMigration_J10002) CLIUtil.getBean(PatchMigration_J10002.class);
			loader.init();
			while (loader.isMoreToProcess()) {
				loader.load();
			}
			logger.info("Load complete. Exiting!!!");
			System.exit(0);
		} catch (Exception e) {
			logger.error("Error loading", e);
			System.exit(1);
		}
	}
	
	@Override
	public void init() throws Exception {
		// Do Nothing
	}

	@Override
	public void execLoad() {
		logger.info("==> MigrationPatch.execLoad()");
		try {
			migrateServicesToNewSchema();
			migratePoliciesToNewSchema();
			updateSequences();
		} catch (Exception e) {
			logger.error("Error whille migrating data.", e);
		}
		logger.info("<== MigrationPatch.execLoad()");
	}

	@Override
	public void printStats() {
		logger.info("Total Number of migrated repositories/services: " + serviceCounter);
		logger.info("Total Number of migrated resources/policies: " + policyCounter);
	}

	public void migrateServicesToNewSchema() throws Exception {
		logger.info("==> MigrationPatch.migrateServicesToNewSchema()");

		try {
			List<XXAsset> repoList = daoMgr.getXXAsset().getAll();

			if (repoList.isEmpty()) {
				return;
			}
			if (!repoList.isEmpty()) {
				EmbeddedServiceDefsUtil.instance().init(svcDBStore);
			}

			svcDBStore.setPopulateExistingBaseFields(true);
			for (XXAsset xAsset : repoList) {

				if (xAsset.getActiveStatus() == AppConstants.STATUS_DELETED) {
					continue;
				}

				RangerService existing = svcDBStore.getServiceByName(xAsset.getName());
				if (existing != null) {
					logger.info("Repository/Service already exists. Ignoring migration of repo: " + xAsset.getName());
					continue;
				}

				RangerService service = new RangerService();
				service = mapXAssetToService(service, xAsset);

				service = svcDBStore.createService(service);

				serviceCounter++;
				logger.info("New Service created. ServiceName: " + service.getName());
			}
			svcDBStore.setPopulateExistingBaseFields(false);
		} catch (Exception e) {
			throw new Exception("Error while migrating data to new Plugin Schema.", e);
		}
		logger.info("<== MigrationPatch.migrateServicesToNewSchema()");
	}

	public void migratePoliciesToNewSchema() throws Exception {
		logger.info("==> MigrationPatch.migratePoliciesToNewSchema()");

		try {
			List<XXResource> resList = daoMgr.getXXResource().getAll();
			if (resList.isEmpty()) {
				return;
			}

			svcDBStore.setPopulateExistingBaseFields(true);
			for (XXResource xRes : resList) {

				if (xRes.getResourceStatus() == AppConstants.STATUS_DELETED) {
					continue;
				}

				XXAsset xAsset = daoMgr.getXXAsset().getById(xRes.getAssetId());
				if (xAsset == null) {
					logger.error("No Repository found for policyName: " + xRes.getPolicyName());
					continue;
				}

				RangerService service = svcDBStore.getServiceByName(xAsset.getName());
				
				if (service == null) {
					logger.error("No Service found for policy. Ignoring migration of such policy, policyName: "
							+ xRes.getPolicyName());
					continue;
				}

				XXPolicy existing = daoMgr.getXXPolicy().findByNameAndServiceId(xRes.getPolicyName(), service.getId());
				if (existing != null) {
					logger.info("Policy already exists. Ignoring migration of policy: " + existing.getName());
					continue;
				}

				RangerPolicy policy = new RangerPolicy();
				policy = mapXResourceToPolicy(policy, xRes, service);

				if(policy != null) {
					policy = svcDBStore.createPolicy(policy);

					policyCounter++;
					logger.info("New policy created. policyName: " + policy.getName());
				}
			}
			svcDBStore.setPopulateExistingBaseFields(false);
		} catch (Exception e) {
			throw new Exception("Error while migrating data to new Plugin Schema.", e);
		}
		logger.info("<== MigrationPatch.migratePoliciesToNewSchema()");
	}

	private RangerService mapXAssetToService(RangerService service, XXAsset xAsset) throws Exception {

		String type = "";
		String name = xAsset.getName();
		String description = xAsset.getDescription();
		Map<String, String> configs = null;

		int typeInt = xAsset.getAssetType();
		XXServiceDef serviceDef = daoMgr.getXXServiceDef().findByName(AppConstants.getLabelFor_AssetType(typeInt).toLowerCase());

		if (serviceDef == null) {
			throw new Exception("No ServiceDefinition found for repository: " + name);
		}
		type = serviceDef.getName();
		configs = jsonUtil.jsonToMap(xAsset.getConfig());

		List<XXServiceConfigDef> mandatoryConfigs = daoMgr.getXXServiceConfigDef().findByServiceDefName(type);
		for (XXServiceConfigDef serviceConf : mandatoryConfigs) {
			if (serviceConf.getIsMandatory()) {
				if (!stringUtil.isEmpty(configs.get(serviceConf.getName()))) {
					continue;
				}
				String dataType = serviceConf.getType();
				String defaultValue = serviceConf.getDefaultvalue();

				if (stringUtil.isEmpty(defaultValue)) {
					defaultValue = getDefaultValueForDataType(dataType);
				}
				configs.put(serviceConf.getName(), defaultValue);
			}
		}

		service.setType(type);
		service.setName(name);
		service.setDescription(description);
		service.setConfigs(configs);

		service.setCreateTime(xAsset.getCreateTime());
		service.setUpdateTime(xAsset.getUpdateTime());

		XXPortalUser createdByUser = daoMgr.getXXPortalUser().getById(xAsset.getAddedByUserId());
		XXPortalUser updByUser = daoMgr.getXXPortalUser().getById(xAsset.getUpdatedByUserId());

		if (createdByUser != null) {
			service.setCreatedBy(createdByUser.getLoginId());
		}
		if (updByUser != null) {
			service.setUpdatedBy(updByUser.getLoginId());
		}
		service.setId(xAsset.getId());

		return service;
	}

	private String getDefaultValueForDataType(String dataType) {

		String defaultValue = "";
		switch (dataType) {
		case "int":
			defaultValue = "0";
			break;
		case "string":
			defaultValue = "unknown";
			break;
		case "bool":
			defaultValue = "false";
			break;
		case "enum":
			defaultValue = "0";
			break;
		case "password":
			defaultValue = "password";
			break;
		default:
			break;
		}
		return defaultValue;
	}

	private RangerPolicy mapXResourceToPolicy(RangerPolicy policy, XXResource xRes, RangerService service) {
		String serviceName = service.getName();
		String serviceType = service.getType();
		String name = xRes.getPolicyName();
		String description = xRes.getDescription();
		Boolean isAuditEnabled = true;
		Boolean isEnabled = true;
		Map<String, RangerPolicyResource> resources = new HashMap<String, RangerPolicyResource>();
		List<RangerPolicyItem> policyItems = new ArrayList<RangerPolicyItem>();

		XXServiceDef svcDef = daoMgr.getXXServiceDef().findByName(serviceType);
		
		if(svcDef == null) {
			logger.error(serviceType + ": service-def not found. Skipping policy '" + name + "'");

			return null;
		}

		List<XXAuditMap> auditMapList = daoMgr.getXXAuditMap().findByResourceId(xRes.getId());
		if (stringUtil.isEmpty(auditMapList)) {
			isAuditEnabled = false;
		}
		if (xRes.getResourceStatus() == AppConstants.STATUS_DISABLED) {
			isEnabled = false;
		}

		Boolean isPathRecursive  = xRes.getIsRecursive() == RangerCommonEnums.BOOL_TRUE;
		Boolean isTableExcludes  = xRes.getTableType() == RangerCommonEnums.POLICY_EXCLUSION;
		Boolean isColumnExcludes = xRes.getColumnType() == RangerCommonEnums.POLICY_EXCLUSION;

		if (StringUtils.equalsIgnoreCase(serviceType, "hdfs")) {
			toRangerResourceList(xRes.getName(), "path", Boolean.FALSE, isPathRecursive, resources);
		} else if (StringUtils.equalsIgnoreCase(serviceType, "hbase")) {
			toRangerResourceList(xRes.getTables(), "table", isTableExcludes, Boolean.FALSE, resources);
			toRangerResourceList(xRes.getColumnFamilies(), "column-family", Boolean.FALSE, Boolean.FALSE, resources);
			toRangerResourceList(xRes.getColumns(), "column", isColumnExcludes, Boolean.FALSE, resources);
		} else if (StringUtils.equalsIgnoreCase(serviceType, "hive")) {
			toRangerResourceList(xRes.getDatabases(), "database", Boolean.FALSE, Boolean.FALSE, resources);
			toRangerResourceList(xRes.getTables(), "table", isTableExcludes, Boolean.FALSE, resources);
			toRangerResourceList(xRes.getColumns(), "column", isColumnExcludes, Boolean.FALSE, resources);
			toRangerResourceList(xRes.getUdfs(), "udf", Boolean.FALSE, Boolean.FALSE, resources);
		} else if (StringUtils.equalsIgnoreCase(serviceType, "knox")) {
			toRangerResourceList(xRes.getTopologies(), "topology", Boolean.FALSE, Boolean.FALSE, resources);
			toRangerResourceList(xRes.getServices(), "service", Boolean.FALSE, Boolean.FALSE, resources);
		} else if (StringUtils.equalsIgnoreCase(serviceType, "storm")) {
			toRangerResourceList(xRes.getTopologies(), "topology", Boolean.FALSE, Boolean.FALSE, resources);
		}

		policyItems = getPolicyItemListForRes(xRes, svcDef);

		policy.setService(serviceName);
		policy.setName(name);
		policy.setDescription(description);
		policy.setIsAuditEnabled(isAuditEnabled);
		policy.setIsEnabled(isEnabled);
		policy.setResources(resources);
		policy.setPolicyItems(policyItems);

		policy.setCreateTime(xRes.getCreateTime());
		policy.setUpdateTime(xRes.getUpdateTime());

		XXPortalUser createdByUser = daoMgr.getXXPortalUser().getById(xRes.getAddedByUserId());
		XXPortalUser updByUser = daoMgr.getXXPortalUser().getById(xRes.getUpdatedByUserId());

		if (createdByUser != null) {
			policy.setCreatedBy(createdByUser.getLoginId());
		}
		if (updByUser != null) {
			policy.setUpdatedBy(updByUser.getLoginId());
		}

		policy.setId(xRes.getId());

		return policy;
	}

	private Map<String, RangerPolicy.RangerPolicyResource> toRangerResourceList(String resourceString, String resourceType, Boolean isExcludes, Boolean isRecursive, Map<String, RangerPolicy.RangerPolicyResource> resources) {
		Map<String, RangerPolicy.RangerPolicyResource> ret = resources == null ? new HashMap<String, RangerPolicy.RangerPolicyResource>() : resources;

		if(StringUtils.isNotBlank(resourceString)) {
			RangerPolicy.RangerPolicyResource resource = ret.get(resourceType);

			if(resource == null) {
				resource = new RangerPolicy.RangerPolicyResource();
				resource.setIsExcludes(isExcludes);
				resource.setIsRecursive(isRecursive);

				ret.put(resourceType, resource);
			}

			Collections.addAll(resource.getValues(), resourceString.split(","));
		}

		return ret;
	}

	private List<RangerPolicyItem> getPolicyItemListForRes(XXResource xRes, XXServiceDef svcDef) {
		List<RangerPolicyItem> policyItems = new ArrayList<RangerPolicyItem>();

		SearchCriteria sc = new SearchCriteria();

		sc.addParam("resourceId", xRes.getId());
		List<VXPermMap> permMapList = xPermMapService.searchXPermMaps(sc).getVXPermMaps();

		HashMap<String, List<VXPermMap>> sortedPermMap = new HashMap<String, List<VXPermMap>>();

		// re-group the list with permGroup as the key
		if (permMapList != null) {
			for(VXPermMap permMap : permMapList) {
				String          permGrp    = permMap.getPermGroup();
				List<VXPermMap> sortedList = sortedPermMap.get(permGrp);

				if(sortedList == null) {
					sortedList = new ArrayList<VXPermMap>();
					sortedPermMap.put(permGrp, sortedList);
				}

				sortedList.add(permMap);
			}
		}

		for (Entry<String, List<VXPermMap>> entry : sortedPermMap.entrySet()) {
			List<String>                 userList   = new ArrayList<String>();
			List<String>                 groupList  = new ArrayList<String>();
			List<RangerPolicyItemAccess> accessList = new ArrayList<RangerPolicyItemAccess>();
			String                       ipAddress  = null;

			RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem();

			for(VXPermMap permMap : entry.getValue()) {
				if(permMap.getPermFor() == AppConstants.XA_PERM_FOR_USER) {
					String userName = getUserName(permMap);

					if (! userList.contains(userName)) {
						userList.add(userName);
					}
				} else if(permMap.getPermFor() == AppConstants.XA_PERM_FOR_GROUP) {
					String groupName = getGroupName(permMap);

					if (! groupList.contains(groupName)) {
						groupList.add(groupName);
					}
				}

				String accessType = ServiceUtil.toAccessType(permMap.getPermType());
				if(StringUtils.isBlank(accessType) || unsupportedLegacyPermTypes.contains(accessType)) {
					logger.info(accessType + ": is not a valid access-type, ignoring accesstype for policy: " + xRes.getPolicyName());
					continue;
				}

				if(StringUtils.equalsIgnoreCase(accessType, "Admin")) {
					policyItem.setDelegateAdmin(Boolean.TRUE);
					if ( svcDef.getId() == EmbeddedServiceDefsUtil.instance().getHBaseServiceDefId()) {
						addAccessType(accessType, accessList);
					}
				} else {
					addAccessType(accessType, accessList);
				}

				ipAddress = permMap.getIpAddress();
			}

			if(CollectionUtils.isEmpty(accessList)) {
				logger.info("no access specified. ignoring policyItem for policy: " + xRes.getPolicyName());
				continue;
			}

			if(CollectionUtils.isEmpty(userList) && CollectionUtils.isEmpty(groupList)) {
				logger.info("no user or group specified. ignoring policyItem for policy: " + xRes.getPolicyName());
				continue;
			}

			policyItem.setUsers(userList);
			policyItem.setGroups(groupList);
			policyItem.setAccesses(accessList);

			if(ipAddress != null && !ipAddress.isEmpty()) {
				XXPolicyConditionDef policyCond = daoMgr.getXXPolicyConditionDef().findByServiceDefIdAndName(svcDef.getId(), "ip-range");

				if(policyCond != null) {
					RangerPolicy.RangerPolicyItemCondition ipCondition = new RangerPolicy.RangerPolicyItemCondition("ip-range", Collections.singletonList(ipAddress));

					policyItem.getConditions().add(ipCondition);
				}
			}

			policyItems.add(policyItem);
		}

		return policyItems;
	}

	private void addAccessType(String accessType, List<RangerPolicyItemAccess> accessList) {
		boolean alreadyExists = false;

		for(RangerPolicyItemAccess access : accessList) {
			if(StringUtils.equalsIgnoreCase(accessType, access.getType())) {
				alreadyExists = true;

				break;
			}
		}

		if(!alreadyExists) {
			accessList.add(new RangerPolicyItemAccess(accessType));
		}
	}

	private void updateSequences() {
		daoMgr.getXXServiceDef().updateSequence();
		daoMgr.getXXService().updateSequence();
		daoMgr.getXXPolicy().updateSequence();
	}

	private String getUserName(VXPermMap permMap) {
		String userName = permMap.getUserName();

		if(userName == null || userName.isEmpty()) {
			Long userId = permMap.getUserId();

			if(userId != null) {
				XXUser xxUser = daoMgr.getXXUser().getById(userId);

				if(xxUser != null) {
					userName = xxUser.getName();
				}
			}
		}

		return userName;
	}

	private String getGroupName(VXPermMap permMap) {
		String groupName = permMap.getGroupName();

		if(groupName == null || groupName.isEmpty()) {
			Long groupId = permMap.getGroupId();

			if(groupId != null) {
				XXGroup xxGroup = daoMgr.getXXGroup().getById(groupId);

				if(xxGroup != null) {
					groupName = xxGroup.getName();
				}
			}
		}

		return groupName;
	}
}