/*
* The Kuali Financial System, a comprehensive financial management system for higher education.
*
* Copyright 2005-2014 The Kuali Foundation
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.kuali.kfs.module.tem.batch.service.impl;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.kuali.kfs.module.tem.TemConstants;
import org.kuali.kfs.module.tem.TemKeyConstants;
import org.kuali.kfs.module.tem.TemConstants.ExpenseImport;
import org.kuali.kfs.module.tem.batch.service.AgencyDataImportService;
import org.kuali.kfs.module.tem.batch.service.DataReportService;
import org.kuali.kfs.module.tem.batch.service.ExpenseImportByTravelerService;
import org.kuali.kfs.module.tem.batch.service.ExpenseImportByTripService;
import org.kuali.kfs.module.tem.businessobject.AgencyImportData;
import org.kuali.kfs.module.tem.businessobject.AgencyStagingData;
import org.kuali.kfs.module.tem.businessobject.defaultvalue.NextAgencyStagingDataIdFinder;
import org.kuali.kfs.module.tem.document.TravelDocument;
import org.kuali.kfs.module.tem.document.service.TravelDocumentService;
import org.kuali.kfs.module.tem.service.TravelExpenseService;
import org.kuali.kfs.sys.KFSPropertyConstants;
import org.kuali.kfs.sys.batch.BatchInputFileType;
import org.kuali.kfs.sys.batch.service.BatchInputFileService;
import org.kuali.kfs.sys.businessobject.GeneralLedgerPendingEntry;
import org.kuali.kfs.sys.businessobject.GeneralLedgerPendingEntrySequenceHelper;
import org.kuali.kfs.sys.context.SpringContext;
import org.kuali.kfs.sys.report.BusinessObjectReportHelper;
import org.kuali.rice.core.api.config.property.ConfigurationService;
import org.kuali.rice.core.api.datetime.DateTimeService;
import org.kuali.rice.krad.service.BusinessObjectService;
import org.kuali.rice.krad.util.ErrorMessage;
import org.kuali.rice.krad.util.ObjectUtils;
import org.springframework.transaction.annotation.Transactional;
public class AgencyDataImportServiceImpl implements AgencyDataImportService {
public static org.apache.log4j.Logger LOG = org.apache.log4j.Logger.getLogger(AgencyDataImportServiceImpl.class);
public final static String REPORT_FILE_NAME_PATTERN = "{0}/{1}_{2}{3}";
private BatchInputFileService batchInputFileService;
private BusinessObjectService businessObjectService;
private ExpenseImportByTravelerService expenseImportByTravelerService;
private ExpenseImportByTripService expenseImportByTripService;
private TravelExpenseService travelExpenseService;
private static ConfigurationService configurationService;
private TravelDocumentService travelDocumentService;
private List<BatchInputFileType> agencyDataImportFileTypes;
private String agencyDataFileErrorDirectory;
private BusinessObjectReportHelper agencyDataTravelerUploadReportHelper;
private BusinessObjectReportHelper agencyDataTripUploadReportHelper;
private BusinessObjectReportHelper agencyDataReconciliationReportHelper;
private DateTimeService dateTimeService;
private DataReportService dataReportService;
private String agencyDataReportDirectory;
private String agencyDataReportFilePrefix;
private String agencyDataReconciliationReportFilePrefix;
/**
* @see org.kuali.kfs.module.tem.batch.service.AgencyDataImportService#importAgencyData()
*/
@Override
public boolean importAgencyData() {
boolean success = true;
LOG.info("Starting Agency Import Process");
for (BatchInputFileType inputFileType : agencyDataImportFileTypes) {
List<String> inputFileNames = batchInputFileService.listInputFileNamesWithDoneFile(inputFileType);
for (String dataFileName : inputFileNames) {
success &= importAgencyDataFile(dataFileName, inputFileType);
}
}
LOG.info("Finished Agency Import Process");
return success;
}
/**
* @see org.kuali.kfs.module.tem.batch.service.AgencyDataImportService#importAgencyDataFile(java.lang.String, org.kuali.kfs.sys.batch.BatchInputFileType)
*/
@Override
public boolean importAgencyDataFile(String dataFileName, BatchInputFileType inputFileType) {
try {
FileInputStream fileContents = new FileInputStream(dataFileName);
byte[] fileByteContent = IOUtils.toByteArray(fileContents);
AgencyImportData agencyData = (AgencyImportData) batchInputFileService.parse(inputFileType, fileByteContent);
IOUtils.closeQuietly(fileContents);
LOG.info("Agency Import - validating: " + dataFileName);
List<AgencyStagingData> validAgencyList = validateAgencyData(agencyData, dataFileName);
boolean isAllValid = validAgencyList.size() == agencyData.getAgencyStagingData().size();
if (!isAllValid) {
String error = "The agency data records to be loaded are rejected due to data problem. Please check the agency data report.";
moveErrorFile(dataFileName, this.getAgencyDataFileErrorDirectory());
}
businessObjectService.save(validAgencyList);
}
catch (Exception ex) {
LOG.error("Failed to process the file : " + dataFileName, ex);
moveErrorFile(dataFileName, this.getAgencyDataFileErrorDirectory());
return false;
}
finally {
// boolean doneFileDeleted = doneFile.delete();
removeDoneFiles(dataFileName);
}
return true;
}
public void moveErrorFile(String dataFileName, String agencyDataFileErrordirectory) {
File dataFile = new File(dataFileName);
if (!dataFile.exists() || !dataFile.canRead()) {
LOG.error("Cannot find/read data file " + dataFileName);
}
else {
try {
FileUtils.moveToDirectory(dataFile, new File(agencyDataFileErrordirectory), true);
}
catch (IOException ex) {
LOG.error("Cannot move the file:" + dataFile + " to the directory: " + agencyDataFileErrordirectory, ex);
}
}
}
/**
* Clears out associated .done files for the processed data files.
*/
protected void removeDoneFiles(String dataFileName) {
File doneFile = new File(StringUtils.substringBeforeLast(dataFileName, ".") + TemConstants.DONE_FILE_SUFFIX);
if (doneFile.exists()) {
doneFile.delete();
}
}
/**
* @see org.kuali.kfs.module.tem.batch.service.AgencyDataImportService#validateAgencyData(org.kuali.kfs.module.tem.businessobject.AgencyImportData, java.lang.String)
*/
@Override
public List<AgencyStagingData> validateAgencyData(AgencyImportData agencyImportData, String dataFileName) {
PrintStream reportDataStream = dataReportService.getReportPrintStream(getAgencyDataReportDirectory(), getAgencyDataReportFilePrefix());
BusinessObjectReportHelper reportHelper = getReportHelper(ExpenseImport.getExpenseImportByCode(agencyImportData.getImportBy()));
HashMap<String, AgencyStagingData> validAgencyStagingDataMap = new HashMap<String, AgencyStagingData>();
try {
dataReportService.writeReportHeader(reportDataStream, dataFileName, TemKeyConstants.MESSAGE_AGENCY_DATA_REPORT_HEADER, reportHelper);
int count = 1;
List<AgencyStagingData> importedAgencyStagingDataList = agencyImportData.getAgencyStagingData();
int listSize = importedAgencyStagingDataList.size();
LOG.info("Validating agency import by traveler: importing "+ listSize +" records");
NextAgencyStagingDataIdFinder idFinder = new NextAgencyStagingDataIdFinder();
for (AgencyStagingData importedAgencyStagingData : importedAgencyStagingDataList) {
String key = null;
importedAgencyStagingData.setId(Integer.valueOf(idFinder.getValue()));
importedAgencyStagingData.setImportBy(agencyImportData.getImportBy());
importedAgencyStagingData.setStagingFileName(StringUtils.substringAfterLast(dataFileName, File.separator));
importedAgencyStagingData.setCreationTimestamp(getDateTimeService().getCurrentTimestamp());
String itineraryData = importedAgencyStagingData.getItineraryDataString();
AgencyStagingData validAgencyStagingData = null;
List<ErrorMessage> errorMessages = new ArrayList<ErrorMessage>();
// validate by Traveler ID
if (importedAgencyStagingData.getExpenseImport() == ExpenseImport.traveler) {
key = importedAgencyStagingData.getTravelerId() + "~" + itineraryData + "~" +
importedAgencyStagingData.getCreditCardOrAgencyCode() + "~" + importedAgencyStagingData.getTransactionPostingDate() + "~" +
importedAgencyStagingData.getTripExpenseAmount() + "~" + importedAgencyStagingData.getTripInvoiceNumber();
LOG.info("Validating agency import by traveler. Record# " + count + " of " + listSize);
if(!validAgencyStagingDataMap.containsKey(key)){
errorMessages.addAll(expenseImportByTravelerService.validateAgencyData(importedAgencyStagingData));
//no errors- load the data
if (errorMessages.isEmpty()) {
validAgencyStagingDataMap.put(key,importedAgencyStagingData);
}
else {
//if there are errors check for required fields missing or duplicate data- load anything else
String errorCode = importedAgencyStagingData.getErrorCode();
if (ObjectUtils.isNotNull(errorCode) &&
!StringUtils.equals(TemConstants.AgencyStagingDataErrorCodes.AGENCY_DUPLICATE_DATA, errorCode)) {
validAgencyStagingDataMap.put(key,importedAgencyStagingData);
}
}
}
else {
// duplicate record found in the file
ErrorMessage error = new ErrorMessage(TemKeyConstants.MESSAGE_AGENCY_DATA_TRAVELER_DUPLICATE_RECORD,
importedAgencyStagingData.getTravelerId(), itineraryData, importedAgencyStagingData.getCreditCardOrAgencyCode(),
importedAgencyStagingData.getTransactionPostingDate().toString(), importedAgencyStagingData.getTripExpenseAmount().toString(), importedAgencyStagingData.getTripInvoiceNumber());
errorMessages.add(error);
}
}
// validate by Trip ID
else if (importedAgencyStagingData.getExpenseImport() == ExpenseImport.trip) {
key = importedAgencyStagingData.getTravelerId() + "~" + importedAgencyStagingData.getTripId() + "~" + importedAgencyStagingData.getCreditCardOrAgencyCode()
+ "~" + importedAgencyStagingData.getTransactionPostingDate() + "~" + importedAgencyStagingData.getTripExpenseAmount() ;
LOG.info("Validating agency import by trip. Record# " + count + " of " + listSize);
if(!validAgencyStagingDataMap.containsKey(key)){
errorMessages.addAll(expenseImportByTripService.validateAgencyData(importedAgencyStagingData));
//no errors- load the data
if (errorMessages.isEmpty()) {
validAgencyStagingDataMap.put(key,importedAgencyStagingData);
}
else {
//if there are errors check for required fields missing or duplicate data- load anything else
String errorCode = importedAgencyStagingData.getErrorCode();
if (ObjectUtils.isNotNull(errorCode) &&
!StringUtils.equals(TemConstants.AgencyStagingDataErrorCodes.AGENCY_DUPLICATE_DATA, errorCode)) {
validAgencyStagingDataMap.put(key,importedAgencyStagingData);
}
}
}
else {
// duplicate record found in the file.
ErrorMessage error = new ErrorMessage(TemKeyConstants.MESSAGE_AGENCY_DATA_TRIP_DUPLICATE_RECORD,
importedAgencyStagingData.getTripId(), importedAgencyStagingData.getCreditCardOrAgencyCode(),
importedAgencyStagingData.getTransactionPostingDate().toString(), importedAgencyStagingData.getTripExpenseAmount().toString(), itineraryData);
errorMessages.add(error);
}
}
//writer to error report
if (!errorMessages.isEmpty()){
dataReportService.writeToReport(reportDataStream, importedAgencyStagingData, errorMessages, reportHelper);
}
count++;
}
}
finally {
if (reportDataStream != null) {
reportDataStream.flush();
reportDataStream.close();
}
}
ArrayList<AgencyStagingData> validAgencyRecords = new ArrayList<AgencyStagingData>() ;
for(Map.Entry<String , AgencyStagingData> entry : validAgencyStagingDataMap.entrySet()) {
validAgencyRecords.add(entry.getValue());
}
return validAgencyRecords;
}
/**
* @see org.kuali.kfs.module.tem.batch.service.AgencyDataImportService#moveAgencyDataToHistoricalExpenseTable()
*/
@Override
public boolean moveAgencyDataToHistoricalExpenseTable() {
LOG.info("Starting Agency Expense Distribution/Reconciliation Process");
List<AgencyStagingData> agencyData = travelExpenseService.retrieveValidAgencyData();
if (ObjectUtils.isNotNull(agencyData) && agencyData.size() > 0) {
PrintStream reportDataStream = dataReportService.getReportPrintStream(getAgencyDataReportDirectory(), getAgencyDataReconciliationReportFilePrefix());
BusinessObjectReportHelper reportHelper = getAgencyDataReconciliationReportHelper();
try {
dataReportService.writeReportHeader(reportDataStream, null, TemKeyConstants.MESSAGE_AGENCY_DATA_RECONCILIATION_REPORT_HEADER, reportHelper);
//set up map for keeping track of sequence helpers per Trip Id. Sequence helper is not needed for ImportBy=TRV
Map<String,GeneralLedgerPendingEntrySequenceHelper> sequenceHelperMap = new HashMap<String,GeneralLedgerPendingEntrySequenceHelper>();
GeneralLedgerPendingEntrySequenceHelper sequenceHelper = null;
for (AgencyStagingData agency : agencyData) {
List<ErrorMessage> errorMessages = new ArrayList<ErrorMessage>();
if (agency.getExpenseImport() == ExpenseImport.trip) {
sequenceHelper = getGeneralLedgerPendingEntrySequenceHelper(agency, sequenceHelperMap);
}
errorMessages.addAll(processAgencyStagingExpense(agency, sequenceHelper));
getBusinessObjectService().save(agency);
//writer to error report
if (!errorMessages.isEmpty()){
dataReportService.writeToReport(reportDataStream, agency, errorMessages, reportHelper);
LOG.info("Agency Data Id: "+ agency.getId() + " was not processed.");
}
else {
LOG.info("Agency Data Id: "+ agency.getId() + " was processed.");
}
}
}
finally {
if (reportDataStream != null) {
reportDataStream.flush();
reportDataStream.close();
}
}
}
LOG.info("Finished Agency Expense Distribution/Reconciliation Process");
return true;
}
/**
* @see org.kuali.kfs.module.tem.batch.service.AgencyDataImportService#processAgencyStagingExpense(org.kuali.kfs.module.tem.businessobject.AgencyStagingData, org.kuali.kfs.sys.businessobject.GeneralLedgerPendingEntrySequenceHelper)
*/
@Transactional
@Override
public List<ErrorMessage> processAgencyStagingExpense(AgencyStagingData agency, GeneralLedgerPendingEntrySequenceHelper sequenceHelper) {
List<ErrorMessage> errors = new ArrayList<ErrorMessage>();
agency.setProcessingTimestamp(getDateTimeService().getCurrentTimestamp());
if (agency.getExpenseImport() == ExpenseImport.traveler) {
errors.addAll(expenseImportByTravelerService.distributeExpense(agency));
}
if (agency.getExpenseImport() == ExpenseImport.trip) {
errors.addAll(expenseImportByTripService.reconciliateExpense(agency, sequenceHelper));
}
return errors;
}
protected GeneralLedgerPendingEntrySequenceHelper getGeneralLedgerPendingEntrySequenceHelper(AgencyStagingData agencyStagingData, Map<String,GeneralLedgerPendingEntrySequenceHelper> sequenceHelperMap) {
String tripId = agencyStagingData.getTripId();
GeneralLedgerPendingEntrySequenceHelper sequenceHelper = sequenceHelperMap.get(tripId);
if (ObjectUtils.isNull(sequenceHelper)) {
Collection<GeneralLedgerPendingEntry> glpes = getGeneralLedgerPendingEntriesForDocumentNumber(agencyStagingData);
if (ObjectUtils.isNotNull(glpes) && !glpes.isEmpty()) {
Integer maxSequenceNumber = 0;
for(GeneralLedgerPendingEntry glpe : glpes) {
Integer sequenceNumber = glpe.getTransactionLedgerEntrySequenceNumber();
maxSequenceNumber = (maxSequenceNumber < sequenceNumber ? sequenceNumber : maxSequenceNumber);
}
maxSequenceNumber += 1;
sequenceHelper = new GeneralLedgerPendingEntrySequenceHelper(maxSequenceNumber);
}
else {
sequenceHelper = new GeneralLedgerPendingEntrySequenceHelper();
}
sequenceHelperMap.put(tripId, sequenceHelper);
}
return sequenceHelper;
}
/**
* Gets the currently existing GLPEs for the document we're going to add GLPEs to
* @param agencyStagingData
* @return
*/
@Override
public Collection<GeneralLedgerPendingEntry> getGeneralLedgerPendingEntriesForDocumentNumber(AgencyStagingData agencyStagingData) {
Collection<GeneralLedgerPendingEntry> glpes = new ArrayList<GeneralLedgerPendingEntry>();
TravelDocument travelDocument = getTravelDocumentService().getParentTravelDocument(agencyStagingData.getTripId());
if (ObjectUtils.isNotNull(travelDocument)) {
Map<String,Object> fieldValues = new HashMap<String, Object>();
fieldValues.put(KFSPropertyConstants.DOCUMENT_NUMBER, travelDocument.getDocumentNumber());
glpes = businessObjectService.findMatching(GeneralLedgerPendingEntry.class, fieldValues);
}
return glpes;
}
protected BusinessObjectReportHelper getReportHelper(ExpenseImport importType){
BusinessObjectReportHelper reportHelper = getAgencyDataTravelerUploadReportHelper();
if(ExpenseImport.traveler == importType){
reportHelper = getAgencyDataTravelerUploadReportHelper();
}
else if(ExpenseImport.trip == importType){
reportHelper = getAgencyDataTripUploadReportHelper();
}
return reportHelper;
}
/**
* Gets the batchInputFileService attribute.
* @return Returns the batchInputFileService.
*/
public BatchInputFileService getBatchInputFileService() {
return batchInputFileService;
}
/**
* Sets the batchInputFileService attribute value.
* @param batchInputFileService The batchInputFileService to set.
*/
public void setBatchInputFileService(BatchInputFileService batchInputFileService) {
this.batchInputFileService = batchInputFileService;
}
/**
* Gets the agencyDataImportFileTypes attribute.
* @return Returns the agencyDataImportFileTypes.
*/
public List<BatchInputFileType> getAgencyDataImportFileTypes() {
return agencyDataImportFileTypes;
}
/**
* Sets the agencyDataImportFileTypes attribute value.
* @param agencyDataImportFileTypes The agencyDataImportFileTypes to set.
*/
public void setAgencyDataImportFileTypes(List<BatchInputFileType> agencyDataImportFileTypes) {
this.agencyDataImportFileTypes = agencyDataImportFileTypes;
}
/**
* Gets the businessObjectService attribute.
* @return Returns the businessObjectService.
*/
public BusinessObjectService getBusinessObjectService() {
return businessObjectService;
}
/**
* Sets the businessObjectService attribute value.
* @param businessObjectService The businessObjectService to set.
*/
public void setBusinessObjectService(BusinessObjectService businessObjectService) {
this.businessObjectService = businessObjectService;
}
/**
* Gets the agencyDataFileErrorDirectory attribute.
* @return Returns the agencyDataFileErrorDirectory.
*/
public String getAgencyDataFileErrorDirectory() {
return agencyDataFileErrorDirectory;
}
/**
* Sets the agencyDataFileErrorDirectory attribute value.
* @param agencyDataFileErrorDirectory The agencyDataFileErrorDirectory to set.
*/
public void setAgencyDataFileErrorDirectory(String agencyDataFileErrorDirectory) {
this.agencyDataFileErrorDirectory = agencyDataFileErrorDirectory;
}
/**
* Gets the expenseImportByTravelerService attribute.
* @return Returns the expenseImportByTravelerService.
*/
public ExpenseImportByTravelerService getExpenseImportByTravelerService() {
return expenseImportByTravelerService;
}
/**
* Sets the expenseImportByTravelerService attribute value.
* @param expenseImportByTravelerService The expenseImportByTravelerService to set.
*/
public void setExpenseImportByTravelerService(ExpenseImportByTravelerService expenseImportByTravelerService) {
this.expenseImportByTravelerService = expenseImportByTravelerService;
}
/**
* Gets the expenseImportByTripService attribute.
* @return Returns the expenseImportByTripService.
*/
public ExpenseImportByTripService getExpenseImportByTripService() {
return expenseImportByTripService;
}
/**
* Sets the expenseImportByTripService attribute value.
* @param expenseImportByTripService The expenseImportByTripService to set.
*/
public void setExpenseImportByTripService(ExpenseImportByTripService expenseImportByTripService) {
this.expenseImportByTripService = expenseImportByTripService;
}
/**
* Gets the travelExpenseService attribute.
* @return Returns the travelExpenseService.
*/
public TravelExpenseService getTravelExpenseService() {
return travelExpenseService;
}
/**
* Sets the travelExpenseService attribute value.
* @param travelExpenseService The travelExpenseService to set.
*/
public void setTravelExpenseService(TravelExpenseService travelExpenseService) {
this.travelExpenseService = travelExpenseService;
}
public BusinessObjectReportHelper getAgencyDataTravelerUploadReportHelper() {
return agencyDataTravelerUploadReportHelper;
}
public void setAgencyDataTravelerUploadReportHelper(BusinessObjectReportHelper agencyDataTravelerUploadReportHelper) {
this.agencyDataTravelerUploadReportHelper = agencyDataTravelerUploadReportHelper;
}
public BusinessObjectReportHelper getAgencyDataTripUploadReportHelper() {
return agencyDataTripUploadReportHelper;
}
public void setAgencyDataTripUploadReportHelper(BusinessObjectReportHelper agencyDataTripUploadReportHelper) {
this.agencyDataTripUploadReportHelper = agencyDataTripUploadReportHelper;
}
public BusinessObjectReportHelper getAgencyDataReconciliationReportHelper() {
return agencyDataReconciliationReportHelper;
}
public void setAgencyDataReconciliationReportHelper(BusinessObjectReportHelper agencyDataReconciliationReportHelper) {
this.agencyDataReconciliationReportHelper = agencyDataReconciliationReportHelper;
}
public DateTimeService getDateTimeService() {
return dateTimeService;
}
public void setDateTimeService(DateTimeService dateTimeService) {
this.dateTimeService = dateTimeService;
}
public String getAgencyDataReportDirectory() {
return agencyDataReportDirectory;
}
public void setAgencyDataReportDirectory(String agencyDataReportDirectory) {
this.agencyDataReportDirectory = agencyDataReportDirectory;
}
public String getAgencyDataReportFilePrefix() {
return agencyDataReportFilePrefix;
}
public void setAgencyDataReportFilePrefix(String agencyDataReportFilePrefix) {
this.agencyDataReportFilePrefix = agencyDataReportFilePrefix;
}
public String getAgencyDataReconciliationReportFilePrefix() {
return agencyDataReconciliationReportFilePrefix;
}
public void setAgencyDataReconciliationReportFilePrefix(String agencyDataReconciliationReportFilePrefix) {
this.agencyDataReconciliationReportFilePrefix = agencyDataReconciliationReportFilePrefix;
}
private static ConfigurationService getConfigurationService() {
if (configurationService == null) {
configurationService = SpringContext.getBean(ConfigurationService.class);
}
return configurationService;
}
public void setDataReportService(DataReportService dataReportService) {
this.dataReportService = dataReportService;
}
public TravelDocumentService getTravelDocumentService() {
return travelDocumentService;
}
public void setTravelDocumentService(TravelDocumentService travelDocumentService) {
this.travelDocumentService = travelDocumentService;
}
}