/*******************************************************************************
* Copyright (c) 2010-2011 VIVO Harvester Team. For full list of contributors, please see the AUTHORS file provided.
* All rights reserved.
* This program and the accompanying materials are made available under the terms of the new BSD license which accompanies this distribution, and is available at http://www.opensource.org/licenses/bsd-license.html
******************************************************************************/
package org.vivoweb.harvester.fetch.nih;
import gov.nih.nlm.ncbi.www.soap.eutils.EFetchPubmedServiceStub;
import gov.nih.nlm.ncbi.www.soap.eutils.EFetchPubmedServiceStub.EFetchResult;
import gov.nih.nlm.ncbi.www.soap.eutils.EFetchPubmedServiceStub.PubmedArticleSet_type0;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.rmi.RemoteException;
import javax.xml.namespace.QName;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import org.apache.axis2.databinding.utils.writer.MTOMAwareXMLSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.vivoweb.harvester.util.InitLog;
import org.vivoweb.harvester.util.args.ArgList;
import org.vivoweb.harvester.util.args.UsageException;
import org.vivoweb.harvester.util.repo.RecordHandler;
import org.vivoweb.harvester.util.repo.XMLRecordOutputStream;
/**
* Module for fetching PubMed Citations using the PubMed SOAP Interface Based on the example code available at the
* PubMed Website.
* @author Stephen V. Williams (swilliams@ctrip.ufl.edu)
* @author Dale R. Scheppler (dscheppler@ctrip.ufl.edu)
* @author Christopher Haines (hainesc@ctrip.ufl.edu)
*/
public class PubmedFetch extends NIHFetch {
/**
* SLF4J Logger
*/
private static Logger log = LoggerFactory.getLogger(PubmedFetch.class);
/**
* The name of the PubMed database
*/
private static String database = "pubmed";
/**
* a base xmlrecordoutputstream
*/
protected static XMLRecordOutputStream baseXMLROS = new XMLRecordOutputStream(new String[]{"PubmedArticle","PubmedBookArticle"}, "<?xml version=\"1.0\"?>\n<!DOCTYPE PubmedArticleSet PUBLIC \"-//NLM//DTD PubMedArticle, 1st January 2011//EN\" \"http://www.ncbi.nlm.nih.gov/entrez/query/DTD/pubmed_110101.dtd\">\n<PubmedArticleSet>\n", "\n</PubmedArticleSet>", ".*?<[pP][mM][iI][dD].*?>(.*?)</[pP][mM][iI][dD]>.*?", null);
/**
* Constructor: Primary method for running a PubMed Fetch. The email address of the person responsible for this
* install of the program is required by NIH guidelines so the person can be contacted if there is a problem, such
* as sending too many queries too quickly.
* @param emailAddress contact email address of the person responsible for this install of the VIVO Harvester
* @param searchTerm query to run on pubmed data
* @param maxRecords maximum number of records to fetch
* @param batchSize number of records to fetch per batch
* @param rh record handler to write to
*/
public PubmedFetch(String emailAddress, String searchTerm, String maxRecords, String batchSize, RecordHandler rh) {
super(emailAddress, searchTerm, maxRecords, batchSize, rh, database);
}
/**
* Constructor
* @param args commandline argument
* @throws IOException error creating task
* @throws UsageException user requested usage message
*/
private PubmedFetch(String[] args) throws IOException, UsageException {
this(getParser("PubmedFetch", database).parse(args));
}
/**
* Constructor
* @param argList parsed argument list
* @throws IOException error creating task
*/
private PubmedFetch(ArgList argList) throws IOException {
super(argList, database);
}
@Override
public void fetchRecords(String WebEnv, String QueryKey, String retStart, String numRecords) throws IOException {
EFetchPubmedServiceStub.EFetchRequest req = new EFetchPubmedServiceStub.EFetchRequest();
req.setQuery_key(QueryKey);
req.setWebEnv(WebEnv);
req.setEmail(getEmailAddress());
req.setTool(getToolName());
req.setRetstart(retStart);
req.setRetmax(numRecords);
int retEnd = Integer.parseInt(retStart) + Integer.parseInt(numRecords);
log.info("Fetching " + retStart + " to " + retEnd + " records from search");
try {
serializeFetchRequest(req);
} catch(RemoteException e) {
throw new IOException("Could not run search", e);
}
}
/**
* Runs, sanitizes, and outputs the results of a EFetch request to the xmlWriter
* <ol>
* <li>create a buffer</li>
* <li>connect to pubmed</li>
* <li>run the efetch request</li>
* <li>get the article set</li>
* <li>create XML writer</li>
* <li>output to buffer</li>
* <li>dump buffer to string</li>
* <li>use sanitizeXML() on string</li>
* </ol>
* @param req the request to run and output results
* @throws IOException Unable to write XML to record
*/
private void serializeFetchRequest(EFetchPubmedServiceStub.EFetchRequest req) throws IOException {
//Create buffer for raw, pre-sanitized output
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
//Connect to pubmed
EFetchPubmedServiceStub service = new EFetchPubmedServiceStub();
//Run the EFetch request
EFetchResult result = service.run_eFetch(req);
//Get the article set
PubmedArticleSet_type0 articleSet = result.getPubmedArticleSet();
XMLStreamWriter writer;
try {
//Create a temporary xml writer to our buffer
writer = XMLOutputFactory.newInstance().createXMLStreamWriter(buffer);
MTOMAwareXMLSerializer serial = new MTOMAwareXMLSerializer(writer);
log.debug("Buffering records");
//Output data
articleSet.serialize(new QName("RemoveMe"), null, serial);
serial.flush();
log.debug("Buffering complete");
log.debug("buffer size: " + buffer.size());
//Dump buffer to String
String iString = buffer.toString("UTF-8");
//Sanitize string (which writes it to xmlWriter)
sanitizeXML(iString);
} catch(XMLStreamException e) {
throw new IOException("Unable to write to output", e);
} catch(UnsupportedEncodingException e) {
throw new IOException("Cannot get xml from buffer", e);
}
}
/**
* Sanitizes XML in preparation for writing to output stream
* <ol>
* <li>Removes xml namespace attributes</li>
* <li>Removes XML wrapper tag</li>
* <li>Splits each record on a new line</li>
* <li>Writes to outputstream writer</li>
* </ol>
* @param strInput The XML to Sanitize.
* @throws IOException Unable to write XML to record
*/
private void sanitizeXML(String strInput) throws IOException {
log.debug("Sanitizing Output");
log.debug("XML File Length - Pre Sanitize: " + strInput.length());
// log.debug("====== PRE-SANITIZE ======\n"+strInput);
String newS = strInput.replaceAll(" xmlns=\".*?\"", "");
newS = newS.replaceAll("</?RemoveMe>", "");
//TODO: this seems really hacky here... revise somehow?
newS = newS.replaceAll("</PubmedArticle>.*?<PubmedArticle", "</PubmedArticle>\n<PubmedArticle");
newS = newS.replaceAll("</PubmedBookArticle>.*?<PubmedBookArticle", "</PubmedBookArticle>\n<PubmedBookArticle");
newS = newS.replaceAll("</PubmedArticle>.*?<PubmedBookArticle", "</PubmedArticle>\n<PubmedBookArticle");
newS = newS.replaceAll("</PubmedBookArticle>.*?<PubmedArticle", "</PubmedBookArticle>\n<PubmedArticle");
log.debug("XML File Length - Post Sanitze: " + newS.length());
// log.debug("====== POST-SANITIZE ======\n"+newS);
log.debug("Sanitization Complete");
log.trace("Writing to output");
// log.debug("buffer contents:\n"+newS);
if(getOsWriter() == null) {
setOs(baseXMLROS.clone().setRso(this));
}
getOsWriter().write(newS);
//file close statements. Warning, not closing the file will leave incomplete xml files and break the translate method
getOsWriter().write("\n");
getOsWriter().flush();
log.trace("Writing complete");
}
@Override
protected int getLatestRecord() throws IOException {
return Integer.parseInt(runESearch("1:8000[dp]", false)[3]);
}
@Override
public void writeRecord(String id, String data) throws IOException {
log.trace("Adding Record "+id);
getRh().addRecord(id, data, getClass());
}
/**
* Main method
* @param args commandline arguments
*/
public static void main(String... args) {
Exception error = null;
try {
InitLog.initLogger(args, getParser("PubmedFetch", database));
log.info("PubmedFetch: Start");
new PubmedFetch(args).execute();
} catch(IllegalArgumentException e) {
log.error(e.getMessage());
log.debug("Stacktrace:",e);
System.out.println(getParser("PubmedFetch", database).getUsage());
error = e;
} catch(UsageException e) {
log.info("Printing Usage:");
System.out.println(getParser("PubmedFetch", database).getUsage());
error = e;
} catch(Exception e) {
log.error(e.getMessage());
log.debug("Stacktrace:",e);
error = e;
} finally {
log.info("PubmedFetch: End");
if(error != null) {
System.exit(1);
}
}
}
}