// Copyright 2015 The Project Buendia Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy
// of the License at: http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distrib-
// uted under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
// OR CONDITIONS OF ANY KIND, either express or implied. See the License for
// specific language governing permissions and limitations under the License.
package org.openmrs.projectbuendia.servlet;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.openmrs.Concept;
import org.openmrs.Encounter;
import org.openmrs.Form;
import org.openmrs.FormField;
import org.openmrs.Obs;
import org.openmrs.Patient;
import org.openmrs.PatientIdentifier;
import org.openmrs.api.EncounterService;
import org.openmrs.api.PatientService;
import org.openmrs.api.context.Context;
import org.openmrs.module.xforms.util.XformsUtil;
import org.openmrs.projectbuendia.ClientConceptNamer;
import org.openmrs.projectbuendia.Utils;
import org.openmrs.projectbuendia.VisitObsValue;
import org.openmrs.projectbuendia.webservices.rest.ChartResource;
import org.openmrs.util.FormUtil;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/** A servlet that generates a CSV dump of all the patient data. */
public class DataExportServlet extends HttpServlet {
protected static Log log = LogFactory.getLog(DataExportServlet.class);
private static final Comparator<Patient> PATIENT_COMPARATOR = new Comparator<Patient>() {
@Override public int compare(Patient p1, Patient p2) {
PatientIdentifier id1 = p1.getPatientIdentifier("MSF");
PatientIdentifier id2 = p2.getPatientIdentifier("MSF");
return Utils.alphanumericComparator.compare(
id1 == null ? null : id1.getIdentifier(),
id2 == null ? null : id2.getIdentifier()
);
}
};
private static final Comparator<Encounter> ENCOUNTER_COMPARATOR = new Comparator<Encounter>() {
@Override public int compare(Encounter e1, Encounter e2) {
return e1.getEncounterDatetime().compareTo(e2.getEncounterDatetime());
}
};
private static final Comparator<Concept> CONCEPT_COMPARATOR = new Comparator<Concept>() {
@Override public int compare(Concept c1, Concept c2) {
return c1.getUuid().compareTo(c2.getUuid());
}
};
private static final String[] FIXED_HEADERS = new String[] {
"Patient UUID",
"MSF patient ID",
"Approximate date of birth",
"Encounter UUID",
"Time in epoch milliseconds",
"Time in ISO8601 UTC",
"Time in yyyy-MM-dd HH:mm:ss UTC",
};
private static final int COLUMNS_PER_OBS = 3;
private static final ClientConceptNamer NAMER = new ClientConceptNamer(Locale.ENGLISH);
public static final int DEFAULT_INTERVAL_MINS = 30;
private final VisitObsValue.ObsValueVisitor stringVisitor =
new VisitObsValue.ObsValueVisitor<String>() {
@Override public String visitCoded(Concept value) {
return NAMER.getClientName(value);
}
@Override public String visitNumeric(Double value) {
return Double.toString(value);
}
@Override public String visitBoolean(Boolean value) {
return Boolean.toString(value);
}
@Override public String visitText(String value) {
return value;
}
@Override public String visitDate(Date d) {
return Utils.YYYYMMDD_UTC_FORMAT.format(d);
}
@Override public String visitDateTime(Date d) {
return Utils.SPREADSHEET_FORMAT.format(d);
}
};
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws
ServletException, IOException {
// Set the default merge mode
boolean merge = true;
// Defines the interval in minutes that will be used to merge encounters.
int interval = DEFAULT_INTERVAL_MINS;
String intervalParameter = request.getParameter("interval");
if (intervalParameter != null) {
int newInterval = Integer.valueOf(intervalParameter);
if (newInterval >= 0) {
interval = newInterval;
if (interval == 0) {
merge = false;
}
} else {
log.error("Interval value is less then 0. Default used.");
}
}
CSVPrinter printer = new CSVPrinter(response.getWriter(), CSVFormat.EXCEL.withDelimiter(','));
//check for authenticated users
if (!XformsUtil.isAuthenticated(request, response, null)) return;
Date now = new Date();
DateFormat format = new SimpleDateFormat("yyyyMMdd_HHmmss");
String filename = String.format("buendiadata_%s.csv", format.format(now));
String contentDispositionHeader = String.format("attachment; filename=%s;", filename);
response.addHeader("Content-Disposition", contentDispositionHeader);
PatientService patientService = Context.getPatientService();
EncounterService encounterService = Context.getEncounterService();
List<Patient> patients = new ArrayList<>(patientService.getAllPatients());
Collections.sort(patients, PATIENT_COMPARATOR);
// We may want to get the observations displayed in the chart/xform, in which case there
// are a few
// sensible orders:
// 1: UUID
// 2: Order in chart
// 3: Order in Xform
// Order in Xform/chart is not good as stuff changes every time we change xform
// So instead we will use UUID order, but use the Chart form to use the concepts to display.
Set<Concept> questionConcepts = new HashSet<>();
for (Form form : ChartResource.getCharts(Context.getFormService())) {
TreeMap<Integer, TreeSet<FormField>> formStructure = FormUtil.getFormStructure(form);
for (FormField groupField : formStructure.get(0)) {
for (FormField fieldInGroup : formStructure.get(groupField.getId())) {
questionConcepts.add(fieldInGroup.getField().getConcept());
}
}
}
FixedSortedConceptIndexer indexer = new FixedSortedConceptIndexer(questionConcepts);
// Write English headers.
writeHeaders(printer, indexer);
Calendar calendar = Calendar.getInstance();
// Loop through all the patients and get their encounters.
for (Patient patient : patients) {
// Define an array that will represent the line that will be inserted in the CSV.
Object[] previousCSVLine = new Object[FIXED_HEADERS.length + indexer.size()*COLUMNS_PER_OBS];
Date deadLine = new Date(0);
ArrayList<Encounter> encounters = new ArrayList<>(
encounterService.getEncountersByPatient(patient));
Collections.sort(encounters, ENCOUNTER_COMPARATOR);
// TODO: For now patients with no encounters are ignored. List them on the future.
if (encounters.size() == 0) continue;
// Loop through all the encounters for this patient to get the observations.
for (Encounter encounter : encounters) {
try {
// Flag to whether we will use the merged version of the encounter
// or the single version.
boolean useMerged = merge;
// Array that will be used to merge in previous encounter with the current one.
Object[] mergedCSVLine = new Object[previousCSVLine.length];
// Duplicate previous encounter into the (future to be) merged one.
System.arraycopy(previousCSVLine, 0, mergedCSVLine, 0, previousCSVLine.length);
// Define the array to be used to store the current encounter.
Object[] currentCSVLine = new Object[FIXED_HEADERS.length + indexer.size()*COLUMNS_PER_OBS];
// If the current encounter is more then "interval" minutes from the previous
// print the previous and reset it.
Date encounterTime = encounter.getEncounterDatetime();
if (encounterTime.after(deadLine)) {
printer.printRecord(previousCSVLine);
previousCSVLine = new Object[FIXED_HEADERS.length + indexer.size()*COLUMNS_PER_OBS];
useMerged = false;
}
// Set the next deadline as the current encounter time plus "interval" minutes.
calendar.setTime(encounterTime);
calendar.add(Calendar.MINUTE, interval);
deadLine = calendar.getTime();
// Fill the fixed columns values.
currentCSVLine[0] = patient.getUuid();
currentCSVLine[1] = patient.getPatientIdentifier("MSF");
if (patient.getBirthdate() != null) {
currentCSVLine[2] = Utils.YYYYMMDD_UTC_FORMAT.format(patient.getBirthdate());
}
currentCSVLine[3] = encounter.getUuid();
currentCSVLine[4] = encounterTime.getTime();
currentCSVLine[5] = Utils.toIso8601(encounterTime);
currentCSVLine[6] = Utils.SPREADSHEET_FORMAT.format(encounterTime);
// All the values fo the fixed columns saved in the current encounter line
// will also be saved to the merged line.
System.arraycopy(currentCSVLine, 0, mergedCSVLine, 0, 7);
// Loop through all the observations for this encounter
for (Obs obs : encounter.getAllObs()) {
Integer index = indexer.getIndex(obs.getConcept());
if (index == null) continue;
// For each observation there are three columns: if the value of the
// observation is a concept, then the three columns contain the English
// name, the OpenMRS ID, and the UUID of the concept; otherwise all
// three columns contain the formatted value.
int valueColumn = FIXED_HEADERS.length + index*COLUMNS_PER_OBS;
// Coded values are treated differently
if (obs.getValueCoded() != null) {
Concept value = obs.getValueCoded();
currentCSVLine[valueColumn] = NAMER.getClientName(value);
currentCSVLine[valueColumn + 1] = value.getId();
currentCSVLine[valueColumn + 2] = value.getUuid();
if (useMerged) {
// If we are still merging the current encounter values into
// the previous one get the previous value and see if it had
// something in it.
String previousValue = (String) mergedCSVLine[valueColumn];
if ((previousValue == null) || (previousValue.isEmpty())) {
// If the previous value was empty copy the current value into it.
mergedCSVLine[valueColumn] = currentCSVLine[valueColumn];
mergedCSVLine[valueColumn + 1] = currentCSVLine[valueColumn + 1];
mergedCSVLine[valueColumn + 2] = currentCSVLine[valueColumn + 2];
} else {
// If the previous encounter have values stored for this
// observation we cannot merge them anymore.
useMerged = false;
}
}
}
// All values except the coded ones will be treated equally.
else {
// Return the value of the the current observation using the visitor.
String value = (String) VisitObsValue.visit(obs, stringVisitor);
// Check if we have values stored for this observation
if ((value != null) && (!value.isEmpty())) {
// Save the value of the observation on the current encounter line.
currentCSVLine[valueColumn] = value;
currentCSVLine[valueColumn + 1] = value;
currentCSVLine[valueColumn + 2] = value;
if (useMerged) {
// Since we are still merging this encounter with the previous
// one let's get the previous value to see if it had something
// stored on it.
String previousValue = (String) mergedCSVLine[valueColumn];
if ((previousValue != null) && (!previousValue.isEmpty())) {
// Yes, we had information stored for this observation on
// the previous encounter
if (obs.getValueText() != null) {
// We only continue merging if the observation is of
// type text, so we concatenate it.
// TODO: add timestamps to the merged values that are of type text
previousValue += "\n" + value;
value = previousValue;
} else {
// Any other type of value we stop the merging.
useMerged = false;
}
}
mergedCSVLine[valueColumn] = value;
mergedCSVLine[valueColumn + 1] = value;
mergedCSVLine[valueColumn + 2] = value;
}
}
}
}
if (useMerged) {
// If after looping through all the observations we didn't had any
// overlapped values we keep the merged line.
previousCSVLine = mergedCSVLine;
} else {
// We had overlapped values so let's print the previous line and make the
// current encounter the previous one. Only if the previous line is not empty.
if (previousCSVLine[0] != null) {
printer.printRecord(previousCSVLine);
}
previousCSVLine = currentCSVLine;
}
} catch (Exception e) {
log.error("Error exporting encounter", e);
}
}
// For the last encounter we print the remaining line.
printer.printRecord(previousCSVLine);
}
}
private void writeHeaders(CSVPrinter printer, FixedSortedConceptIndexer indexer) throws
IOException {
for (String fixedHeader : FIXED_HEADERS) {
printer.print(fixedHeader);
}
for (int i = 0; i < indexer.size(); i++) {
// For each observation there are three columns: one for the English
// name, one for the OpenMRS ID, and one for the UUID of the concept.
assert COLUMNS_PER_OBS == 3;
Concept concept = indexer.getConcept(i);
printer.print(NAMER.getClientName(concept));
printer.print(concept.getId());
printer.print(concept.getUuid());
}
printer.println();
}
/** Indexes a fixed set of concepts in sorted UUID order. */
private static class FixedSortedConceptIndexer {
final Concept[] concepts;
public FixedSortedConceptIndexer(Collection<Concept> concepts) {
this.concepts = concepts.toArray(new Concept[concepts.size()]);
Arrays.sort(this.concepts, CONCEPT_COMPARATOR);
}
public Integer getIndex(Concept concept) {
int index = Arrays.binarySearch(concepts, concept, CONCEPT_COMPARATOR);
if (index < 0) return null;
return index;
}
public Concept getConcept(int i) {
return concepts[i];
}
public int size() {
return concepts.length;
}
}
}