/*
Copyright (C) 2005-2012, by the President and Fellows of Harvard College.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Dataverse Network - A web application to share, preserve and analyze research data.
Developed at the Institute for Quantitative Social Science, Harvard University.
Version 3.0.
*/
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package edu.harvard.iq.dvn.ingest.dsb.impl;
import edu.harvard.iq.dvn.ingest.dsb.*;
import java.io.*;
import static java.lang.System.*;
import java.util.*;
import java.util.logging.*;
import java.lang.reflect.*;
import java.util.regex.*;
import org.rosuda.REngine.*;
import org.rosuda.REngine.Rserve.*;
import org.apache.commons.lang.*;
import org.apache.commons.lang.builder.*;
/**
*
* @author asone
*/
public class DvnRforeignFileConversionServiceImpl{
// ----------------------------------------------------- static filelds
private static Logger dbgLog = Logger.getLogger(DvnRforeignFileConversionServiceImpl.class.getPackage().getName());
public static String DSB_TMP_DIR=null;
private static String TMP_DATA_FILE_NAME = "susetfile4Rjob";
public static String TMP_DATA_FILE_EXT =".tab";
private static String RSERVE_HOST = null;
private static String RSERVE_USER = null;
private static String RSERVE_PWD = null;
private static int RSERVE_PORT;
private static String DSB_HOST_PORT= null;
public static String DSB_CTXT_DIR =null;
private static Map<String, String> dwlndParam = new HashMap<String, String>();
private static Map<String, Method> runMethods = new HashMap<String, Method>();
private static String regexForRunMethods = "^run(\\w+)Request$" ;
private static String RESULT_DIR_PREFIX = "Zlg_";
public static String RWRKSP_FILE_PREFIX = "dvnDataFramefile_";
public static String dtflprefix = "dvnDataFile_";
public static String TEMP_DIR = System.getProperty("java.io.tmpdir");
static {
DSB_TMP_DIR = System.getProperty("vdc.dsb.temp.dir");
if (DSB_TMP_DIR == null){
DSB_TMP_DIR = "/tmp/VDC/DSB";
}
RSERVE_HOST = System.getProperty("vdc.dsb.host");
if (RSERVE_HOST == null){
//RSERVE_HOST= "dsb-2.hmdc.harvard.edu";
RSERVE_HOST= "vdc-build.hmdc.harvard.edu";
}
DSB_HOST_PORT = System.getProperty("vdc.dsb.port");
if (DSB_HOST_PORT == null){
DSB_HOST_PORT= "80";
}
RSERVE_USER = System.getProperty("vdc.dsb.rserve.user");
if (RSERVE_USER == null){
RSERVE_USER= "rserve";
}
RSERVE_PWD = System.getProperty("vdc.dsb.rserve.pwrd");
if (RSERVE_PWD == null){
RSERVE_PWD= "rserve";
}
if (System.getProperty("vdc.dsb.rserve.port") == null ){
RSERVE_PORT= 6311;
} else {
RSERVE_PORT = Integer.parseInt(System.getProperty("vdc.dsb.rserve.port"));
}
// various constants: option-wise
// download formats
dwlndParam.put("D01","dat");
dwlndParam.put("D02","ssc");
dwlndParam.put("D03","dta");
dwlndParam.put("D04","RData");
Pattern p = Pattern.compile(regexForRunMethods);
for (Method m: DvnRforeignFileConversionServiceImpl.class.getDeclaredMethods()){
Matcher mtr = p.matcher(m.getName());
if (mtr.matches()){
runMethods.put(mtr.group(1), m);
}
}
}
static String VDC_R_STARTUP_FILE="vdc_startup.R";
static String librarySetup = "source(paste(.libPaths()[1], '/../share/dvn/" + VDC_R_STARTUP_FILE + "', sep = ''));";
boolean DEBUG = true;
// ----------------------------------------------------- instance filelds
public String PID = null;
public String tempFileName = null;
public String tempFileNameNew = null;
public String wrkdir = null;
public String requestdir = null;
public List<String> replicationFile = new LinkedList<String>();
// ----------------------------------------------------- constructor
public DvnRforeignFileConversionServiceImpl(){
dbgLog.fine("***** DvnRforeignFileConversionServiceImpl: within the constructor starts here *****");
// initialization
PID = RandomStringUtils.randomNumeric(6);
requestdir = "Zlg_" + PID;
wrkdir = DSB_TMP_DIR + "/" + requestdir;
tempFileName = DSB_TMP_DIR + "/" + TMP_DATA_FILE_NAME
+"." + PID + TMP_DATA_FILE_EXT;
tempFileNameNew = wrkdir + "/" + TMP_DATA_FILE_NAME
+"." + PID + TMP_DATA_FILE_EXT;
dbgLog.fine("requestdir="+requestdir);
dbgLog.fine("wrkdir="+wrkdir);
dbgLog.fine("tempFileName="+tempFileName);
dbgLog.fine("tempFileNameNew="+tempFileNameNew);
}
public void setupWorkingDirectory(RConnection c){
try{
// set up the working directory:
String checkWrkDir = "if (!file_test('-d', '"+wrkdir+"')) {dir.create('"+wrkdir+"', showWarnings = FALSE, recursive = TRUE);}";
dbgLog.fine("w permission="+checkWrkDir);
c.voidEval(checkWrkDir);
} catch (RserveException rse) {
rse.printStackTrace();
}
}
/** *************************************************************
* Execute an R-based dvn statistical analysis request
*
* @param sro a DvnRJobRequest object that contains various parameters
* @return a Map that contains various information about results
*/
public Map<String, String> execute(DvnRJobRequest sro) {
dbgLog.fine("***** DvnRforeignFileConversionServiceImpl: execute() starts here *****");
// set the return object
Map<String, String> result = new HashMap<String, String>();
// temporary result
Map<String, String> tmpResult = new HashMap<String, String>();
try {
// Set up an Rserve connection
dbgLog.fine("sro dump:\n"+ToStringBuilder.reflectionToString(sro, ToStringStyle.MULTI_LINE_STYLE));
dbgLog.fine("RSERVE_USER="+RSERVE_USER+"[default=rserve]");
dbgLog.fine("RSERVE_PWD="+RSERVE_PWD+"[default=rserve]");
dbgLog.fine("RSERVE_PORT="+RSERVE_PORT+"[default=6311]");
RConnection c = new RConnection(RSERVE_HOST, RSERVE_PORT);
dbgLog.fine("hostname="+RSERVE_HOST);
c.login(RSERVE_USER, RSERVE_PWD);
dbgLog.fine(">" + c.eval("R.version$version.string").asString() + "<");
// check working directories
// This needs to be done *before* we try to create any files
// there!
setupWorkingDirectory(c);
// save the data file at the Rserve side
String infile = sro.getSubsetFileName();
InputStream inb = new BufferedInputStream(
new FileInputStream(infile));
int bufsize;
byte[] bffr = new byte[1024];
RFileOutputStream os =
c.createFile(tempFileName);
while ((bufsize = inb.read(bffr)) != -1) {
os.write(bffr, 0, bufsize);
}
os.close();
inb.close();
// Rserve code starts here
dbgLog.fine("wrkdir="+wrkdir);
c.voidEval(librarySetup);
// variable type
/*
vartyp <-c(1,1,1)
*/
// java side
// int [] jvartyp = {1,1,1};// = mp.get("vartyp").toArray()
// int [] jvartyp = sro.getVariableTypes();
/*
StringBuilder sb = new StringBuilder();
for (int i = 0 ; i< jvartyp.length; i++){
if (i == (jvartyp.length -1)){
sb.append(String.valueOf(jvartyp[i]));
} else {
sb.append(String.valueOf(jvartyp[i])+", ");
}
}
// R side
*/
/*
* Note that we want to use the "getVariableTypesWithBoolean method
* below; when we create a SRO object for analysis, in
* DvnRDataAnalysisServiceImpl, we'll still be using getVariableTypes
* method, that don't recognize Booleans as a distinct class of
* variables. So those would be treated simply as numeric categoricals
* (factors) with the "TRUE" and "FALSE" labels. But for the purposes
* of saving the subset in R format, we want to convert these into
* R "logical" vectors.
*/
dbgLog.fine("raw variable type="+sro.getVariableTypesWithBoolean());
c.assign("vartyp", new REXPInteger(sro.getVariableTypesWithBoolean()));
String [] tmpt = c.eval("vartyp").asStrings();
dbgLog.fine("vartyp length="+ tmpt.length + "\t " +
StringUtils.join(tmpt,","));
// variable format (date/time)
/*
varFmt<-list();
c.voidEval("varFmt<-list()");
*/
Map<String, String> tmpFmt = sro.getVariableFormats();
dbgLog.fine("tmpFmt="+tmpFmt);
if (tmpFmt != null){
Set<String> vfkeys = tmpFmt.keySet();
String[] tmpfk = (String[]) vfkeys.toArray(new String[vfkeys.size()]);
String[] tmpfv = getValueSet(tmpFmt, tmpfk);
c.assign("tmpfk", new REXPString(tmpfk));
c.assign("tmpfv", new REXPString(tmpfv));
String fmtNamesLine = "names(tmpfv)<- tmpfk";
c.voidEval(fmtNamesLine);
String fmtValuesLine ="varFmt<- as.list(tmpfv)";
c.voidEval(fmtValuesLine);
} else {
String [] varFmtN ={};
List<String> varFmtV = new ArrayList<String>();
c.assign("varFmt", new REXPList(new RList(varFmtV, varFmtN)));
}
/*
vnames<-c("race","age","vote")
*/
// variable names
// String [] jvnames = {"race","age","vote"};
String [] jvnamesRaw = sro.getVariableNames();
String [] jvnames = null;
//VariableNameFilterForR nf = new VariableNameFilterForR(jvnamesRaw);
if (sro.hasUnsafedVariableNames){
// create list
jvnames = sro.safeVarNames;
dbgLog.fine("renamed="+StringUtils.join(jvnames,","));
} else {
jvnames = jvnamesRaw;
}
String vnQList = DvnDSButil.joinNelementsPerLine(jvnames,true);
c.assign("vnames", new REXPString(jvnames));
// confirmation
String [] tmpjvnames = c.eval("vnames").asStrings();
dbgLog.fine("vnames:"+ StringUtils.join(tmpjvnames, ","));
/*
x<-read.table141vdc(file="/tmp/VDC/t.28948.1.tab",
col.names=vnames, colClassesx=vartyp, varFormat=varFmt)
*/
//String datafilename = "/nfs/home/A/asone/java/rcode/t.28948.1.tab";
// tab-delimited file name = tempFileName
// Parameters:
// file -> tempFileName
// col.names -> Arrays.deepToString(new REXPString(jvnames)).asStrings())
// colClassesx -> Arrays.deepToString((new REXPInteger(sro.getVariableTypes())).asStrings())
// varFormat -> Arrays.deepToString((new REXPString(getValueSet(tmpFmt, tmpFmt.keySet().toArray(new String[tmpFmt.keySet().size()])))).asStrings())
dbgLog.fine("<<<<<<<<<<<<<<<<<<<<<<<<<");
dbgLog.fine("col.names = " + Arrays.deepToString((new REXPString(jvnames)).asStrings()));
dbgLog.fine("colClassesx = " + Arrays.deepToString((new REXPInteger(sro.getVariableTypesWithBoolean())).asStrings()));
dbgLog.fine("varFormat = " + Arrays.deepToString((new REXPString(getValueSet(tmpFmt, tmpFmt.keySet().toArray(new String[tmpFmt.keySet().size()])))).asStrings()));
dbgLog.fine(">>>>>>>>>>>>>>>>>>>>>>>>>");
String readtableline = "x<-read.table141vdc(file='"+tempFileName+
"', col.names=vnames, colClassesx=vartyp, varFormat=varFmt )";
dbgLog.fine("readtable="+readtableline);
c.voidEval(readtableline);
// safe-to-raw variable name
/*
attr(x, "Rsafe2raw")<-list();
*/
//if (nf.hasRenamedVariables()){
if (sro.hasUnsafedVariableNames){
dbgLog.fine("unsafeVariableNames exist");
// create list
//jvnames = nf.getFilteredVarNames();
jvnames = sro.safeVarNames;
String[] rawNameSet = sro.renamedVariableArray;
String[] safeNameSet = sro.renamedResultArray;
c.assign("tmpRN", new REXPString(rawNameSet));
c.assign("tmpSN", new REXPString(safeNameSet));
String raw2safevarNameTableLine = "names(tmpRN)<- tmpSN";
c.voidEval(raw2safevarNameTableLine);
String attrRsafe2rawLine = "attr(x, 'Rsafe2raw')<- as.list(tmpRN)";
c.voidEval(attrRsafe2rawLine);
} else {
String attrRsafe2rawLine = "attr(x, 'Rsafe2raw')<-list();";
c.voidEval(attrRsafe2rawLine);
}
//Map<String, String> Rsafe2raw = sro.getRaw2SafeVarNameTable();
// asIs
/*
for (i in 1:dim(x)[2]){if (attr(x,"var.type")[i] == 0) {
x[[i]]<-I(x[[i]]); x[[i]][ x[[i]] == '' ]<-NA }}
*/
String asIsline = "for (i in 1:dim(x)[2]){ "+
"if (attr(x,'var.type')[i] == 0) {" +
"x[[i]]<-I(x[[i]]); x[[i]][ x[[i]] == '' ]<-NA }}";
c.voidEval(asIsline);
// replication: copy the data.frame
String repDVN_Xdupline = "dvnData<-x";
c.voidEval(repDVN_Xdupline);
tmpResult.put("dvn_dataframe", "dvnData");
// subsetting (mutating the data.frame strips non-default attributes
//
// variable type must be re-attached
//String varTypeNew = "vartyp<-c(" + StringUtils.join( sro.getUpdatedVariableTypesAsString(),",")+")";
// c.voidEval(varTypeNew);
dbgLog.fine("updated var Type ="+ sro.getUpdatedVariableTypes());
c.assign("vartyp", new REXPInteger(sro.getUpdatedVariableTypes()));
String reattachVarTypeLine = "attr(x, 'var.type') <- vartyp";
c.voidEval(reattachVarTypeLine);
// TODO:
// "getUpdatedVariableTypesWithBooleans" ?? -- L.A.
// replication: variable type
String repDVN_vt = "attr(dvnData, 'var.type') <- vartyp";
c.voidEval(repDVN_vt);
// variable Id
/*
attr(x, "var.nmbr")<-c("v198057","v198059","v198060")
*/
// String[] jvarnmbr = {"v198057","v198059","v198060"};
//String[] jvarnmbr = sro.getVariableIds();
// after recoding
String[] jvarnmbr = sro.getUpdatedVariableIds();
String viQList = DvnDSButil.joinNelementsPerLine(jvarnmbr,true);
c.assign("varnmbr",new REXPString(jvarnmbr));
String attrVarNmbrLine = "attr(x, 'var.nmbr')<-varnmbr";
c.voidEval(attrVarNmbrLine);
// confirmation
String [] vno = c.eval("attr(x, 'var.nmbr')").asStrings();
dbgLog.fine("varNo="+StringUtils.join(vno, ","));
// replication: variable number
String repDVN_vn = "attr(dvnData, 'var.nmbr') <- varnmbr";
c.voidEval(repDVN_vn);
// variable labels
/*
attr(x, "var.labels")<-c("race","age","vote")
*/
// String[] jvarlabels = {"race","age","vote"};
// String[] jvarlabels = sro.getVariableLabels();
// after recoding
String[] jvarlabels = sro.getUpdatedVariableLabels();
String vlQList = DvnDSButil.joinNelementsPerLine(jvarlabels,true);
c.assign("varlabels", new REXPString(jvarlabels));
String attrVarLabelsLine = "attr(x, 'var.labels')<-varlabels";
c.voidEval(attrVarLabelsLine);
// confirmation
String [] vlbl = c.eval("attr(x, 'var.labels')").asStrings();
dbgLog.fine("varlabels="+StringUtils.join(vlbl, ","));
// replication:
String repDVN_vl = "attr(dvnData, 'var.labels') <- varlabels";
c.voidEval(repDVN_vl);
// --------- start: block to be used for the production code
// value-label table
/*
VALTABLE<-list()
VALTABLE[["1"]]<-list(
"2"="white",
"1"="others")
attr(x, 'val.table')<-VALTABLE
*/
// create the VALTABLE
String vtFirstLine = "VALTABLE<-list()";
c.voidEval(vtFirstLine);
// vltbl includes both base and recoded cases when it was generated
Map<String, Map<String, String>> vltbl = sro.getValueTable();
Map<String, String> rnm2vi = sro.getRawVarNameToVarIdTable();
String[] updatedVariableIds = sro.getUpdatedVariableIds();
//for (int j=0;j<jvnamesRaw.length;j++){
for (int j=0;j<updatedVariableIds.length;j++){
// if this variable has its value-label table,
// pass its key and value arrays to the Rserve
// and finalize a value-table at the Rserve
//String varId = rnm2vi.get(jvnamesRaw[j]);
String varId = updatedVariableIds[j];
if (vltbl.containsKey(varId)){
Map<String, String> tmp = (HashMap<String, String>)vltbl.get(varId);
Set<String> vlkeys = tmp.keySet();
String[] tmpk = (String[]) vlkeys.toArray(new String[vlkeys.size()]);
String[] tmpv = getValueSet(tmp, tmpk);
// debug
dbgLog.fine("tmp:k="+ StringUtils.join(tmpk,","));
dbgLog.fine("tmp:v="+ StringUtils.join(tmpv,","));
// index number starts from 1(not 0)
int indx = j +1;
dbgLog.fine("index="+indx);
if (tmpv.length > 0){
c.assign("tmpk", new REXPString(tmpk));
c.assign("tmpv", new REXPString(tmpv));
String namesValueLine = "names(tmpv)<- tmpk";
c.voidEval(namesValueLine);
String sbvl = "VALTABLE[['"+ Integer.toString(indx)+"']]" + "<- as.list(tmpv)";
dbgLog.fine("frag="+sbvl);
c.voidEval(sbvl);
// confirmation test for j-th variable name
REXP jl = c.parseAndEval(sbvl);
dbgLog.fine("jl("+j+") = "+jl);
}
}
}
// debug: confirmation test for value-table
dbgLog.fine("length of vl="+ c.eval("length(VALTABLE)").asInteger());
String attrValTableLine = "attr(x, 'val.table')<-VALTABLE";
c.voidEval(attrValTableLine);
// replication: value-label table
String repDVN_vlt = "attr(dvnData, 'val.table') <- VALTABLE";
c.voidEval(repDVN_vlt);
// --------- end: block to be used for the production code
// missing-value list: TO DO
/*
MSVLTBL<-list(); attr(x, 'missval.table')<-MSVLTBL
*/
String msvStartLine = "MSVLTBL<-list();";
c.voidEval(msvStartLine);
// data structure
String attrMissvalLine = "attr(x, 'missval.table')<-MSVLTBL";
c.voidEval(attrMissvalLine);
// replication: missing value table
String repDVN_mvlt = "attr(dvnData, 'missval.table') <- MSVLTBL";
c.voidEval(repDVN_mvlt);
// attach attributes(tables) to the data.frame
/*
x<-createvalindex(dtfrm=x, attrname='val.index')
x<-createvalindex(dtfrm=x, attrname='missval.index')
*/
String createVIndexLine = "x<-createvalindex(dtfrm=x, attrname='val.index');";
c.voidEval(createVIndexLine);
String createMVIndexLine = "x<-createvalindex(dtfrm=x, attrname='missval.index');";
c.voidEval(createMVIndexLine);
// reflection block: start ------------------------------------------>
String requestTypeToken = sro.getRequestType();// (Download|EDA|Xtab|Zelig)
dbgLog.fine("requestTypeToken="+requestTypeToken);
// get a test method
Method mthd = runMethods.get(requestTypeToken);
dbgLog.fine("method="+mthd);
try {
// invoke this method
result = (Map<String, String>) mthd.invoke(this, sro, c);
} catch (InvocationTargetException e) {
//Throwable cause = e.getCause();
//err.format(cause.getMessage());
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
}
// add the variable list
result.put("variableList", joinNelementsPerLine(jvnamesRaw, 5,
null, false, null, null));
//result.put("variableList",StringUtils.join(jvnamesRaw, ", "));
// replication: var-level unf
String repDVN_varUNF = "attr(dvnData, 'variableUNF') <- paste(unf(dvnData,version=3))";
c.voidEval(repDVN_varUNF);
// calculate the file-leve UNF
String fileUNFline = "fileUNF <- paste(summary(unf(dvnData, version=3)))";
c.voidEval(fileUNFline);
String fileUNF = c.eval("fileUNF").asString();
if (fileUNF == null){
fileUNF = "NA";
}
// replication: file-level unf
String repDVN_fileUNF = "attr(dvnData, 'fileUNF') <- fileUNF";
c.voidEval(repDVN_fileUNF);
String RversionLine = "R.Version()$version.string";
String Rversion = c.eval(RversionLine).asString();
// replication: R version
String repDVN_Rversion = "attr(dvnData, 'R.version') <- R.Version()$version.string";
c.voidEval(repDVN_Rversion);
String zeligVersionLine = "packageDescription('Zelig')$Version";
String zeligVersion = c.eval(zeligVersionLine).asString();
//String RexecDate = c.eval("date()").asString();
String RexecDate = c.eval("as.character(as.POSIXct(Sys.time()))").asString();
// replication: date
String repDVN_date = "attr(dvnData, 'date') <- as.character(as.POSIXct(Sys.time()))";
c.voidEval(repDVN_date);
String repDVN_dfOrigin =
"attr(dvnData, 'data.frame.origin')<- " +
"list('provider'='Dataverse Network Project'," +
"'format' = list('name' = 'DVN data.frame', 'version'='1.3'))";
c.voidEval(repDVN_dfOrigin);
/*
if (result.containsKey("option")){
result.put("R_run_status", "T");
} else {
result.put("option", requestTypeToken.toLowerCase()); //download zelig eda xtab
result.put("R_run_status", "F");
}
*/
if (sro.hasRecodedVariables()){
if (sro.getSubsetConditions() != null){
result.put("subsettingCriteria", StringUtils.join(sro.getSubsetConditions(),"\n"));
// replication: subset lines
String[] sbst = null;
sbst = (String[])sro.getSubsetConditions().toArray(new String[sro.getSubsetConditions().size()]);
c.assign("sbstLines", new REXPString(sbst));
String repDVN_sbst = "attr(dvnData, 'subsetLines') <- sbstLines";
c.voidEval(repDVN_sbst);
}
/* to be used in the future? */
if (sro.getRecodeConditions() != null){
result.put("recodingCriteria", StringUtils.join(sro.getRecodeConditions(),"\n"));
String[] rcd = null;
rcd = (String[])sro.getRecodeConditions().toArray(new String[sro.getRecodeConditions().size()]);
c.assign("rcdtLines", new REXPString(rcd));
String repDVN_rcd = "attr(dvnData, 'recodeLines') <- rcdtLines";
c.voidEval(repDVN_rcd);
}
}
// save workspace as a replication data set
String RdataFileName = "DVNdataFrame."+PID+".RData";
result.put("Rdata", "/"+requestdir+ "/" + RdataFileName);
String saveWS = "save('dvnData', file='"+ wrkdir +"/"+ RdataFileName +"')";
dbgLog.fine("save the workspace="+saveWS);
c.voidEval(saveWS);
// write back the R workspace to the dvn
String wrkspFileName = wrkdir +"/"+ RdataFileName;
dbgLog.fine("wrkspFileName="+wrkspFileName);
int wrkspflSize = getFileSize(c,wrkspFileName);
File wsfl = writeBackFileToDvn(c, wrkspFileName, RWRKSP_FILE_PREFIX,"RData", wrkspflSize);
result.put("dvn_RData_FileName",wsfl.getName());
if (wsfl != null){
result.put("wrkspFileName", wsfl.getAbsolutePath());
dbgLog.fine("wrkspFileName="+wsfl.getAbsolutePath());
} else {
dbgLog.fine("wrkspFileName is null");
}
result.put("library_1","VDCutil");
result.put("fileUNF",fileUNF);
result.put("dsbHost", RSERVE_HOST);
result.put("dsbPort", DSB_HOST_PORT);
result.put("dsbContextRootDir", DSB_CTXT_DIR );
result.put("PID", PID);
result.put("Rversion", Rversion);
result.put("zeligVersion", zeligVersion);
result.put("RexecDate", RexecDate);
result.putAll(tmpResult);
dbgLog.fine("result object (before closing the Rserve):\n"+result);
// reflection block: end
// close the Rserve connection
c.close();
} catch (RserveException rse) {
// RserveException (Rserve is not running)
rse.printStackTrace();
result.put("dsbContextRootDir", DSB_CTXT_DIR );
result.put("PID", PID);
result.put("option",sro.getRequestType().toLowerCase());
result.put("RexecError", "true");
return result;
} catch (REXPMismatchException mme) {
// REXP mismatch exception (what we got differs from what we expected)
mme.printStackTrace();
result.put("dsbContextRootDir", DSB_CTXT_DIR );
result.put("PID", PID);
result.put("option",sro.getRequestType().toLowerCase());
result.put("RexecError", "true");
return result;
} catch (IOException ie){
ie.printStackTrace();
result.put("dsbContextRootDir", DSB_CTXT_DIR );
result.put("PID", PID);
result.put("option",sro.getRequestType().toLowerCase());
result.put("RexecError", "true");
return result;
} catch (Exception ex){
ex.printStackTrace();
result.put("dsbContextRootDir", DSB_CTXT_DIR );
result.put("PID", PID);
result.put("option",sro.getRequestType().toLowerCase());
result.put("RexecError", "true");
return result;
}
return result;
}
// ----------------------------------------------------- utilitiy methods
/**
* Returns the array of values that corresponds the order of
* provided keys
*
* @param
* @return
*/
public static String[] getValueSet(Map<String, String> mp, String[] keys) {
List<String> tmpvl = new ArrayList<String>();
for (int i=0; i< keys.length; i++){
tmpvl.add(mp.get(keys[i]));
}
String[] tmpv = (String[])tmpvl.toArray(new String[tmpvl.size()]);
return tmpv;
}
/** *************************************************************
* Handles a downloading request
*
* @param
* @return
*/
public Map<String, String> runDownloadRequest(DvnRJobRequest sro, RConnection c){
String optionBanner = "########### downloading option ###########\n";
dbgLog.fine(optionBanner);
Map<String, String> sr = new HashMap<String, String>();
sr.put("requestdir", requestdir);
sr.put("option", "download");
String RcodeFile = "Rcode." + PID + ".R";
try {
// create a temp dir
String createTmpDir = "dir.create('"+wrkdir +"')";
dbgLog.fine("createTmpDir="+createTmpDir);
c.voidEval(createTmpDir);
String dwnldOpt = sro.getDownloadRequestParameter();
String dwnldFlxt = dwlndParam.get(dwnldOpt);
sr.put("format", dwnldFlxt);
String dataFileName = "Data." + PID + "." + dwnldFlxt;
sr.put("subsetfile", "/"+requestdir+ "/" +dataFileName);
// data file to be copied back to the dvn
String dsnprfx = wrkdir + "/" + dataFileName;
String univarDataDwnld = "univarDataDwnld(dtfrm=x,"+
"dwnldoptn='"+dwnldOpt+"'"+
", dsnprfx='"+dsnprfx+"')";
dbgLog.fine("univarDataDwnld="+univarDataDwnld);
c.voidEval(univarDataDwnld);
int wbFileSize = getFileSize(c,dsnprfx);
dbgLog.fine("wbFileSize="+wbFileSize);
// write back the data file to the dvn
File dtfl = writeBackFileToDvn(c, dsnprfx, dtflprefix, dwlndParam.get(dwnldOpt), wbFileSize);
if (dtfl != null){
sr.put("wbDataFileName", dtfl.getAbsolutePath());
dbgLog.fine("wbDataFileName="+dtfl.getAbsolutePath());
} else {
dbgLog.fine("wbDataFileName is null");
}
// tab data file
String mvTmpTabFile = "file.rename('"+ tempFileName +"','"+ tempFileNameNew +"')";
c.voidEval(mvTmpTabFile);
dbgLog.fine("move temp file="+mvTmpTabFile);
// move the temp dir to the web-temp root dir
// String mvTmpDir = "file.rename('"+wrkdir+"','"+webwrkdir+"')";
// dbgLog.fine("web-temp_dir="+mvTmpDir);
// c.voidEval(mvTmpDir);
} catch (RserveException rse) {
rse.printStackTrace();
sr.put("RexecError", "true");
return sr;
}
sr.put("RexecError", "false");
return sr;
}
/** *************************************************************
*
*
* @param
* @return
*/
public String joinNelementsPerLine(String[] vn, int divisor){
String vnl = null;
if (vn.length < divisor){
vnl = StringUtils.join(vn, ", ");
} else {
StringBuilder sb = new StringBuilder();
int iter = vn.length / divisor;
int lastN = vn.length % divisor;
if (lastN != 0){
iter++;
}
int iterm = iter - 1;
for (int i= 0; i<iter; i++){
int terminalN = divisor;
if ((i == iterm ) && (lastN != 0)){
terminalN = lastN;
}
for (int j = 0; j< terminalN; j++){
if ( (divisor*i +j +1) == vn.length){
sb.append(vn[j + i*divisor]);
} else {
sb.append(vn[j + i*divisor] + ", ");
}
}
sb.append("\n");
}
vnl = sb.toString();
dbgLog.fine(vnl);
}
return vnl;
}
/** *************************************************************
*
*
* @param
* @return
*/
public String joinNelementsPerLine(String[] vn, int divisor, String sp,
boolean quote, String qm, String lnsp){
if (!(divisor >= 1)){
divisor = 1;
} else if ( divisor > vn.length) {
divisor = vn.length;
}
String sep = null;
if (sp != null){
sep = sp;
} else {
sep = ", ";
}
String qmrk = null;
if (quote){
if (qm == null){
qmrk = ",";
} else {
if (qm.equals("\"")){
qmrk = "\"";
} else {
qmrk = qm;
}
}
} else {
qmrk = "";
}
String lineSep = null;
if (lnsp == null){
lineSep = "\n";
} else {
lineSep = lnsp;
}
String vnl = null;
if (vn.length < divisor){
vnl = StringUtils.join(vn, sep);
} else {
StringBuilder sb = new StringBuilder();
int iter = vn.length / divisor;
int lastN = vn.length % divisor;
if (lastN != 0){
iter++;
}
int iterm = iter - 1;
for (int i= 0; i<iter; i++){
int terminalN = divisor;
if ((i == iterm ) && (lastN != 0)){
terminalN = lastN;
}
for (int j = 0; j< terminalN; j++){
if ( (divisor*i +j +1) == vn.length){
sb.append(qmrk + vn[j + i*divisor] + qmrk);
} else {
sb.append(qmrk + vn[j + i*divisor] + qmrk + sep);
}
}
if (i < (iter-1)){
sb.append(lineSep);
}
}
vnl = sb.toString();
dbgLog.fine("results:\n"+vnl);
}
return vnl;
}
/** *************************************************************
*
*
* @param
* @return
*/
public File writeBackFileToDvn(RConnection c, String targetFilename,
String tmpFilePrefix, String tmpFileExt, int fileSize){
// set up a temp file
File tmprsltfl = null;
String resultFile = tmpFilePrefix + PID + "." + tmpFileExt;
//String resultFilePrefix = tmpFilePrefix + PID + "_";
//String rfsffx = "." + tmpFileExt;
RFileInputStream ris = null;
OutputStream outbr = null;
try {
//tmprsltfl = File.createTempFile(resultFilePrefix, rfsffx);
tmprsltfl = new File(TEMP_DIR, resultFile);
//outbr = new FileOutputStream(tmprsltfl);
outbr = new BufferedOutputStream(new FileOutputStream(tmprsltfl));
//File tmp = new File(targetFilename);
//long tmpsize = tmp.length();
// open the input stream
ris = c.openFile(targetFilename);
if (fileSize < 1024*1024*500){
int bfsize = fileSize;
byte[] obuf = new byte[bfsize];
ris.read(obuf);
//while ((obufsize =)) != -1) {
outbr.write(obuf, 0, bfsize);
//}
}
ris.close();
outbr.close();
return tmprsltfl;
} catch (FileNotFoundException fe){
fe.printStackTrace();
dbgLog.fine("FileNotFound exception occurred");
return tmprsltfl;
} catch (IOException ie){
ie.printStackTrace();
dbgLog.fine("IO exception occurred");
} finally {
if (ris != null){
try {
ris.close();
} catch (IOException e){
}
}
if (outbr != null){
try {
outbr.close();
} catch (IOException e){
}
}
}
return tmprsltfl;
}
/** *************************************************************
*
*
* @param
* @return
*/
public int getFileSize(RConnection c, String targetFilename){
dbgLog.fine("targetFilename="+targetFilename);
int fileSize = 0;
try {
String fileSizeLine = "round(file.info('"+targetFilename+"')$size)";
fileSize = c.eval(fileSizeLine).asInteger();
} catch (RserveException rse) {
rse.printStackTrace();
} catch (REXPMismatchException mme) {
mme.printStackTrace();
}
return fileSize;
}
}