ReportServiceImpl.java

package com.transformuk.bdt.service;

import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import javax.servlet.ServletContext;
import javax.servlet.http.HttpSession;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;

import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.transformuk.bdt.domain.Bfsf;
import com.transformuk.bdt.domain.CrawledContent;
import com.transformuk.bdt.domain.Lep;
import com.transformuk.bdt.domain.Options;
import com.transformuk.bdt.domain.Page;
import com.transformuk.bdt.domain.Questions;
import com.transformuk.bdt.mapit.domain.MapItPartialPostCode;
import com.transformuk.bdt.model.OtherSources;
import com.transformuk.bdt.model.ProfileQuestions;
import com.transformuk.bdt.model.ReportModel;
import com.transformuk.bdt.util.GenericHelper;
import com.transformuk.bdt.util.OtherSourcesComparator;
import com.transformuk.bdt.util.StaticReferences;

@Service
@Configuration
@PropertySource(value = "classpath:conf.properties")
public class ReportServiceImpl implements IReportService {

	private static final String TITLE_SUFFIX = "-title";

	@Autowired
	MapItServiceImpl mapitService;

	@Autowired
	LepServiceImpl lepService;

	@Autowired
	BfsfServiceImpl bfsfService;
	
	@Autowired
	ExternalBfsfServiceImpl externalBfsfService;

	@Autowired
	QuestionServiceImpl questionService;
	
	@Autowired
	CrawlerService crawlerService;

	@Value("${suffix.keywords}")
	private String suffixKeywords;
	
	@Value("${es.crawlerqueries.type}")
	private String esCrawlerQueryType;
	
	@Value("${es.indexname}")
	private String esIndex;
	
	@Value("${es.bfsf.type}")
	private String esBfsfType;
	
	@Value("${es.bfsf.external.type}")
	private String esExternalType;
	
	@Value("${es.crawler.result.size}")
	private String crawlerResultSize;
	
	@Value("${es.crawler.indexname}")
	private String esCrawlerIndex;
	
	@Value("${es.crawler.type}")
	private String esCrawlerType;

	private static final Logger LOGGER = Logger.getLogger(ReportServiceImpl.class);

	@Override
	public ReportModel buildDataForReport(ProfileQuestions profileQuestions, 
										HttpSession session) throws Exception {

		try {
			String postCode = profileQuestions.getValues().get(
					StaticReferences.QC_POST_CODE);

			ReportModel reportModel = populateReportModel(session, profileQuestions);
			reportModel.setPostCode(postCode);

			LOGGER.debug("Fetching mapit response for postcode:" + postCode);
			
			boolean isPartialPostCode =  (Boolean)session.getAttribute(StaticReferences.IS_PARTIAL_POSTCODE);
			ResponseEntity<String> mapItResponse = getMapItDataByCode(postCode, isPartialPostCode);

			String jsonResponse = mapItResponse.getBody();
			
			if (isPartialPostCode) {
				//if partial post code use another API
				jsonResponse = getJsonResponseByPointAPI(jsonResponse);
			}
			LOGGER.debug("Mapit json response for postcode= " + jsonResponse);
			
			Map<String, List<String>> gssNameResponseMap = mapitService
					.getGssNamesFromPostCodeResponse(jsonResponse, isPartialPostCode);

			// build lep names/phone data from gss codes
			List<String> gssList = gssNameResponseMap.get(StaticReferences.GSS);
			reportModel.setLepResult(buildLepData(gssList));

			// set if HG based on country name
			List<String> countryNameList = gssNameResponseMap.get(StaticReferences.COUNTRY_NAME);
			if (!countryNameList.isEmpty()) {
				if (StringUtils.equalsIgnoreCase(StaticReferences.ENGLAND, countryNameList.get(0))) {
					reportModel.setBasedInEngland(true);
				} else {
					setAsNotHighGrowth(session);
				}
			}

			// use council code in query, should only contain ONE value
			List<String> councilCodeList = gssNameResponseMap.get(StaticReferences.COUNCIL_CODE);
			if (!councilCodeList.isEmpty()) {
				// remove known suffixes
				councilCodeList.set(0, GenericHelper.suffixRemover(councilCodeList.get(0), suffixKeywords));
				gssNameResponseMap.put(StaticReferences.COUNCIL_CODE, councilCodeList);
			}

			String council = gssNameResponseMap.get(StaticReferences.COUNCIL_CODE).get(0);
			String region = getRegionfromEUR(gssNameResponseMap);
			
			//query bfsf data
			Map<String, List<Bfsf>> bfsfSearchResults = doBfsfSearch(profileQuestions, session, council, region, bfsfService);
			reportModel.setSearchResult(bfsfSearchResults);
			
			// build query for crawled data
			Map<String, Set<String>> queryTermsForCrawledData = buildQueryTermsByCategoriesForCrawledData(profileQuestions, session);
			//query crawler
			Map<String, HashSet<CrawledContent>> crawledResults = doCrawledSearch(queryTermsForCrawledData);
			
			//store results from all external i.e. other sources
			Map<String, List<OtherSources>> otherSourcesMap = new LinkedHashMap<String, List<OtherSources>>();
			addCrawlerResultToMap(crawledResults, otherSourcesMap);

			//query External (greatbusiness) data
			bfsfSearchResults = doBfsfSearch(profileQuestions, session, council, region, externalBfsfService);
			addExtBfsfResultToMap(bfsfSearchResults, otherSourcesMap);
			
			//sort them on their score
			sortCrawlerAndExtBfsfResults(otherSourcesMap);
			
			//set the data for report
			reportModel.setOtherSources(otherSourcesMap);
			
			return reportModel;
		} catch (Exception e) {
			LOGGER.error("Exception building report: " + e.getMessage());
			throw e;
		}
	}

	private void sortCrawlerAndExtBfsfResults(Map<String, List<OtherSources>> otherSourcesMap) {
		for (Entry<String, List<OtherSources>> entry : otherSourcesMap.entrySet()) {
			//sort the data
			Collections.sort(entry.getValue(),new OtherSourcesComparator());
		}
	}

	private void addExtBfsfResultToMap(Map<String, List<Bfsf>> bfsfSearchResults,
			Map<String, List<OtherSources>> otherSourcesMap) {
		
		for (Entry<String, List<Bfsf>> entry : bfsfSearchResults.entrySet()) {
			List<OtherSources> otherSourcesList = new ArrayList<OtherSources>();
			for (Bfsf bfsf : entry.getValue()) {
				OtherSources otherSources = new OtherSources();
				otherSources.setDescription(bfsf.getShort_description());
				otherSources.setScore(bfsf.get_score());
				otherSources.setTitle(bfsf.getTitle());
				otherSources.setUrl(bfsf.getWeb_url());

				//only add if this source not already added before
				//across ALL categories of 'Other Sources' 
				boolean isDuplicate = isSourceRecordDuplicate(otherSources, otherSourcesMap);
				if (!isDuplicate)
					otherSourcesList.add(otherSources);
			}
			//add to existing list if already created by crawler
			if (otherSourcesMap.get(entry.getKey()) != null) 
				otherSourcesMap.get(entry.getKey()).addAll(otherSourcesList);
			else
				otherSourcesMap.put(entry.getKey(),otherSourcesList);
		}
	}
	
	private boolean isSourceRecordDuplicate(OtherSources newOtherSources, Map<String, List<OtherSources>> otherSourcesMap) {
		
		//compares the url of new and existing sources to check if duplicate
		for (Entry<String, List<OtherSources>> entry : otherSourcesMap.entrySet()) {
			for (OtherSources existingOtherSources : entry.getValue()) {
				String existingSourceUrl = existingOtherSources.getUrl();
				String newSourceUrl = newOtherSources.getUrl();
				
				if ( (StringUtils.isNotBlank(existingSourceUrl) 
		    			&& (StringUtils.isNotBlank(newSourceUrl)))
		    			&& (existingSourceUrl.equalsIgnoreCase(newSourceUrl))) {
					return true;
				}
			}
		}
		return false;
		
	}
	private void addCrawlerResultToMap(Map<String, HashSet<CrawledContent>> crawledResults,
			Map<String, List<OtherSources>> otherSourcesMap) {
		
		for (Entry<String, HashSet<CrawledContent>> entry : crawledResults.entrySet()) {
			List<OtherSources> otherSourcesList = new ArrayList<OtherSources>();
			for (CrawledContent crawledContent : entry.getValue()) {
				OtherSources otherSources = new OtherSources();
				otherSources.setContent(crawledContent.getContent());
				otherSources.setDescription(crawledContent.getMetatagDescription());
				otherSources.setScore(crawledContent.get_score());
				otherSources.setTitle(crawledContent.getTitle());
				otherSources.setUrl(crawledContent.getUrl());
				otherSourcesList.add(otherSources);
			}
			otherSourcesMap.put(entry.getKey(), otherSourcesList);
		}
	}

	private String getJsonResponseByPointAPI(String jsonResponse)
			throws JsonParseException, JsonMappingException, IOException {
		
			MapItPartialPostCode mapItPostCodeResponse = mapitService.convertDataFromPartialPostCode(jsonResponse);
			
			//make call to the point API
			ResponseEntity<String> jsonDataPointApi = mapitService.getDataFromPointApi(mapItPostCodeResponse);
			return jsonDataPointApi.getBody();
	}

	private ResponseEntity<String> getMapItDataByCode(String postCode, boolean isPartialPostCode) {
		
		ResponseEntity<String> mapItResponse = null;
		if (isPartialPostCode) {
			mapItResponse = mapitService.getMapItDataForPartialPostCode(postCode);
		}
		else {
			mapItResponse = mapitService.getMapItDataForFullPostCode(postCode);
		}
		
		if (mapItResponse.getStatusCode() != HttpStatus.OK) {
			LOGGER.error("MapIt postcode lookup returned HttpResponse error code: "
					+ mapItResponse.getStatusCode());
			
			throw new RuntimeException("MapIt Service error,http code: "+ mapItResponse.getStatusCode());
		}
		return mapItResponse;
	}

	@SuppressWarnings("unchecked")
	private Map<String, List<Bfsf>> doBfsfSearch(ProfileQuestions profileQuestions,
			HttpSession session, String councilCode, String region,
			ICommonBfsfService commonBfsfService)
			throws Exception {

		Map<String, String> categoryMap = (Map<String, String>) session
				.getAttribute(StaticReferences.QUERY_CATEGORIES);

		// hold results
		Map<String, List<Bfsf>> searchResultMap = new LinkedHashMap<String, List<Bfsf>>();

		boolean isFinanceQuery = false;
		String financeType = (String) session
				.getAttribute(StaticReferences.QUERY_FINANCE);
		if (StringUtils.isNotEmpty(financeType))
			isFinanceQuery = true;

		// iterate over all categories chosen by user
		for (Map.Entry<String, String> entry : categoryMap.entrySet()) {

			boolean isCurrCatFinance = StringUtils.startsWithIgnoreCase(
					entry.getKey(), StaticReferences.CATEGORY_FINANCE);

			String query = getBfsfQueryByUserInput(session, isFinanceQuery,
					isCurrCatFinance);
			query = query
					.replace(StaticReferences.CATEGORIES, entry.getValue())
					.replace(StaticReferences.COUNCIL, councilCode)
					.replace(
							StaticReferences.STAGE,
							(String) session
									.getAttribute(StaticReferences.QUERY_STAGE))
					.replace(
							StaticReferences.SIZE,
							(String) session
									.getAttribute(StaticReferences.QUERY_SIZE))
					.replace(
							StaticReferences.SECTOR,
							(String) session
									.getAttribute(StaticReferences.QUERY_SECTOR))
					.replace(StaticReferences.REGION, region);

			if (isFinanceQuery && isCurrCatFinance) {
				// substitute finance
				query = query.replace(StaticReferences.FINANCE_TYPE,
						financeType);
			}

			LOGGER.debug("Firing query for BFSF :" + query);

			// fire query and store results
			List<Bfsf> resultList = commonBfsfService.searchExtOrBfsfBySource( query, searchResultMap.values());
			if (!CollectionUtils.isEmpty(resultList)) {
				searchResultMap.put(entry.getKey(), resultList);
			}

		}
		return searchResultMap;
	}

	private Map<String, HashSet<CrawledContent>> doCrawledSearch(Map<String, Set<String>> queryTermsForCrawledData) throws Exception {
		
		// hold results
		Map<String, HashSet<CrawledContent>> searchResultMap = new LinkedHashMap<String, HashSet<CrawledContent>>();
				
		QueryBuilder qb = QueryBuilders.matchAllQuery();
		String crawlerQuery = crawlerService.getCrawlerQueryByQueryBuilder(
								qb, 0, esIndex, esCrawlerQueryType);
		
		for (Map.Entry<String, Set<String>> entry : queryTermsForCrawledData.entrySet()) {
			StringBuilder queryBuilder = new StringBuilder();
			queryBuilder.append("[");
			//static data i.e. {"match": {"url": {"query": "gov.uk","type": "phrase","boost": 0.5}}},
			//queryBuilder.append("{\"match\": { \"url\": {\"query\": \"gov.uk\",\"type\": \"phrase\",\"boost\": 0.5}}},");
			
			Set<String> tagSet = entry.getValue();
			int tagSize = 0;
			
			for (String tag : tagSet) {
				tagSize++;
				queryBuilder.append("{\"match\": { \"matchedtags\": {\"query\":");
				queryBuilder.append("\"").append(tag).append(TITLE_SUFFIX).append("\",");
				queryBuilder.append("\"type\": \"phrase\"}}}");
				if (tagSize != tagSet.size())
					queryBuilder.append(",");
			}
			
			queryBuilder.append("]");
			
			//substitute the dynamic bits of query
			String query = crawlerQuery.replace("\"%DYNAMIC_TAGS%\"", queryBuilder.toString());
			
			//make actual query
			LOGGER.debug("Firing query for Crawled Data:"+query);
			HashSet<CrawledContent> resultList = crawlerService.searchCrawledContentBySource(
							query, crawlerResultSize, esCrawlerIndex, esCrawlerType, searchResultMap.values());
			
			if (!CollectionUtils.isEmpty(resultList)) {
				searchResultMap.put(entry.getKey(), resultList);
			}
			
		}
		
		return searchResultMap;
		
	}
	
	private String getRegionfromEUR(Map<String, List<String>> gssNameResponseMap) {
		String eurCode = gssNameResponseMap.get(StaticReferences.NAMES).get(0);
		if (StringUtils.equalsIgnoreCase(eurCode, StaticReferences.EASTERN))
			return StaticReferences.EAST_OF_ENGLAND;

		return eurCode;
	}

	private List<Lep> buildLepData(List<String> gssList) throws Exception {

		List<Lep> lepDataList = new ArrayList<Lep>();
		for (String gssCode : gssList) {
			List<Lep> lepListForGSSCode = lepService
					.getMatchingLepFromGssCode(gssCode);
			// note for same gsscode we can get multiple LEPS therefore add all list
			lepDataList.addAll(lepListForGSSCode);
		}
		return lepDataList;
	}

	private ReportModel populateReportModel(HttpSession session,
			ProfileQuestions profileQuestions) {
		// reportModel partially already build by controller
		ReportModel reportModel = (ReportModel) session
				.getAttribute(StaticReferences.REPORT_MODEL);

		reportModel.setHighGrowth((Boolean) session
				.getAttribute(StaticReferences.HIGH_GROWTH));

		String supportAreas = profileQuestions.getValues().get(
				StaticReferences.QC_SUPP_ACT);
		if (StringUtils.containsIgnoreCase(supportAreas,
				StaticReferences.SUPPAREA_EXPORT)) {
			reportModel.setExportArea(true);
		}

		String sector = profileQuestions.getValues().get(
				StaticReferences.QC_IND_DD);
		if (StringUtils.equalsIgnoreCase(sector,
				StaticReferences.SECTOR_MANUFACTURING)) {
			reportModel.setManufacturingSector(true);
		}
		return reportModel;
	}

	private String getBfsfQueryByUserInput(HttpSession session,
			boolean isFinanceQuery, boolean isCurrCatFinance) throws Exception {

		// get the bfsf query based on users category selection
		ServletContext servletContext = session.getServletContext();

		if (isFinanceQuery && isCurrCatFinance) {
			// if the query is already stored in the context
			String questionQueryFinance = (String) servletContext
					.getAttribute(StaticReferences.QUESTION_QUERY_WITH_FINANCE);
			if (questionQueryFinance == null) {

				QueryBuilder qb = QueryBuilders.boolQuery().must(
						matchPhraseQuery("support_types",
								StaticReferences.FINANCE_TYPE));

				// fire query
				questionQueryFinance = bfsfService
						.getBfsfQueryByQueryBuilder(qb);

				// set it in context for reuse
				servletContext.setAttribute(
						StaticReferences.QUESTION_QUERY_WITH_FINANCE,
						questionQueryFinance);
			}
			return questionQueryFinance;
		} else {
			String questionQueryNonFinance = (String) servletContext
					.getAttribute(StaticReferences.QUESTION_QUERY_NON_FINANCE);
			if (questionQueryNonFinance == null) {

				QueryBuilder qb = QueryBuilders.boolQuery().mustNot(
						matchPhraseQuery("support_types",
								StaticReferences.FINANCE_TYPE));

				// fire query
				questionQueryNonFinance = bfsfService.getBfsfQueryByQueryBuilder(qb);

				// set it in context
				servletContext.setAttribute(
						StaticReferences.QUESTION_QUERY_NON_FINANCE,
						questionQueryNonFinance);
			}
			return questionQueryNonFinance;
		}
	}

	@SuppressWarnings("unchecked")
	private Map<String, Set<String>> buildQueryTermsByCategoriesForCrawledData(
			ProfileQuestions profileQuestions, HttpSession session) throws Exception {
		
		//ensure unique elements i.e. set
		Map<String, Set<String>> categoryTagsMap = new LinkedHashMap<String, Set<String>>();
		
		List<String> categoryList = (List<String>) session.getAttribute(StaticReferences.CATEGORY_PAGES);
		for (String catPageName : categoryList) {
			Set<String> termsCrawledList = new LinkedHashSet<String>();
			Page page = getPageDetailsByName(catPageName);
			if (page.isFireQuery()) {
				for (Questions question : page.getQuestions()) {
					// add all tags from the questionTag
					termsCrawledList.addAll(question.getQuestionTags());

					// add all tags from the answer responseTag
					termsCrawledList.addAll(buildTermsFromUsersSelectedOption(
							question, profileQuestions));
				}
				categoryTagsMap.put(page.getPageTitle(), termsCrawledList);
			}
		}
		return categoryTagsMap;
	}

	protected Set<String> buildTermsFromUsersSelectedOption(Questions q,
			ProfileQuestions profileQuestions) {

		String userAnswerCode = profileQuestions.getValues().get(
				q.getQuestionCode());
		String[] userAnswersList = StringUtils.split(userAnswerCode, ',');
		//only add unique values
		Set<String> responseTagList = new HashSet<String>();
		for (String answerCode : userAnswersList) {
			Options userOption = getOptionByAnswerCode(q, answerCode);
			responseTagList.addAll(userOption.getResponseTags());
		}
		return responseTagList;
	}

	// returns Option pojo
	protected Options getOptionByAnswerCode(Questions q, String userAnswerCode) {
		if (q.getOptions() != null) {
			for (Options option : q.getOptions()) {
				if (StringUtils.equals(option.getAnswerCode(), userAnswerCode)) {
					return option;
				}
			}
		}
		return null;
	}

	private Page getPageDetailsByName(String pageName) throws Exception {
		QueryBuilder qb = QueryBuilders.matchPhraseQuery(
				StaticReferences.QUESTION_QUERY, pageName);
		return questionService.searchQuestionsByQueryBuilder(qb).get(0);
	}

	private void setAsNotHighGrowth(HttpSession session) {
		Boolean isHighGrowthBusiness = (Boolean) session
				.getAttribute(StaticReferences.HIGH_GROWTH);
		if (isHighGrowthBusiness) {
			// only set it to false as by default it is HG
			session.setAttribute(StaticReferences.HIGH_GROWTH, false);
		}
	}

}