////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2009-2013 Denim Group, Ltd.
//
// The contents of this file are subject to the Mozilla Public License
// Version 2.0 (the "License"); you may not use this file except in
// compliance with the License. You may obtain a copy of the License at
// http://www.mozilla.org/MPL/
//
// Software distributed under the License is distributed on an "AS IS"
// basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
// License for the specific language governing rights and limitations
// under the License.
//
// The Original Code is ThreadFix.
//
// The Initial Developer of the Original Code is Denim Group, Ltd.
// Portions created by Denim Group, Ltd. are Copyright (C)
// Denim Group, Ltd. All Rights Reserved.
//
// Contributor(s): Denim Group, Ltd.
//
////////////////////////////////////////////////////////////////////////
package com.denimgroup.threadfix.service.channel;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipFile;
import org.apache.commons.io.IOUtils;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import org.springframework.beans.factory.annotation.Autowired;
import com.denimgroup.threadfix.data.dao.ChannelSeverityDao;
import com.denimgroup.threadfix.data.dao.ChannelTypeDao;
import com.denimgroup.threadfix.data.dao.ChannelVulnerabilityDao;
import com.denimgroup.threadfix.data.entities.ChannelSeverity;
import com.denimgroup.threadfix.data.entities.ChannelType;
import com.denimgroup.threadfix.data.entities.ChannelVulnerability;
import com.denimgroup.threadfix.data.entities.Finding;
import com.denimgroup.threadfix.data.entities.Scan;
import com.denimgroup.threadfix.data.entities.SurfaceLocation;
import com.denimgroup.threadfix.webapp.controller.ScanCheckResultBean;
/**
* Parses the Skipfish output file. The zip upload will go look at the relevant request.dat file and try to
* parse the correct parameter out, but relies on the fact that the Skipfish
* payload is this string: -->">'>'"< in order to grab the variable names.
*
* @author mcollins
*
*/
public class SkipfishChannelImporter extends AbstractChannelImporter {
private String folderName;
// TODO use a different method to grab parameters.
// This one attempts to parse based on a limited set of payloads.
private static final String [] SKIPFISH_PAYLOADS = { "--\\x3e\\x22\\x3e\\x27\\x3e\\x27\\x22",
"\\x3e\\x27\\x3e\\x22\\x3e\\x3",
"./",
".\\",
"'\"",
"\\x27\\x22",
"-->\">'>'\"<",
"%3B%3F"};
private static final String [] SKIPFISH_PAYLOAD_REGEXES = { "--\\\\x3e\\\\x22\\\\x3e\\\\x27\\\\x3e\\\\x27\\\\x22",
"\\\\x3e\\\\x27\\\\x3e\\\\x22\\\\x3e\\\\x3",
"\\.",
"\\.",
"'\"",
"\\\\x27\\\\x22",
"-->\\\">'>'\\\"<",
"%3B%3F"};
private static final String REGEX_START = "[\\?\\&]([0-9a-zA-Z_\\-]+)=[^\\&]+";
private static final String INTERESTING_FILE_CODE = "40401";
private static final String DIRECTORY_LISTING = "Directory listing";
private Calendar date;
/**
* Constructor.
*
* @param channelTypeDao
* Spring dependency.
* @param channelVulnerabilityDao
* Spring dependency.
* @param channelSeverityDao
* Spring dependency.
* @param vulnerabilityMapLogDao
* Spring dependency.
*/
@Autowired
public SkipfishChannelImporter(ChannelTypeDao channelTypeDao,
ChannelVulnerabilityDao channelVulnerabilityDao,
ChannelSeverityDao channelSeverityDao) {
this.channelTypeDao = channelTypeDao;
this.channelVulnerabilityDao = channelVulnerabilityDao;
this.channelSeverityDao = channelSeverityDao;
setChannelType(ChannelType.SKIPFISH);
}
/*
* (non-Javadoc)
*
* @see
* com.denimgroup.threadfix.service.channel.ChannelImporter#parseInput()
*/
@Override
public Scan parseInput() {
InputStream samplesFileStream = getSampleFileInputStream();
List<?> map = null;
map = getArrayFromSamplesFile(samplesFileStream);
try {
samplesFileStream.close();
} catch (IOException e) {
log.warn("The Skipfish samples.js fileStream wouldn't close.", e);
}
if (map == null)
return null;
List<Finding> findings = getFindingsFromMap(map);
Scan scan = new Scan();
scan.setFindings(findings);
scan.setApplicationChannel(applicationChannel);
scan.setImportTime(date);
deleteZipFile();
deleteScanFile();
return scan;
}
private InputStream getSampleFileInputStream() {
if (inputStream == null)
return null;
zipFile = unpackZipStream();
if (zipFile == null)
return null;
folderName = findFolderName(zipFile);
InputStream samplesFileStream = null;
if (folderName != null)
samplesFileStream = getFileFromZip(folderName + "/samples.js");
else
samplesFileStream = getFileFromZip("samples.js");
return samplesFileStream;
}
// This method parses the examples.js file into a Java object using a JSON parser.
private List<?> getArrayFromSamplesFile(
InputStream sampleFileInputStream) {
if (sampleFileInputStream == null)
return null;
BufferedReader reader = new BufferedReader(new InputStreamReader(
new DataInputStream(sampleFileInputStream)));
String issuesString = "[";
String tempString = null;
boolean write = false;
try {
StringBuffer buffer = new StringBuffer();
while ((tempString = reader.readLine()) != null) {
if (write)
buffer.append(tempString.replace("'",
"\"").replace("\\", "\\\\"));
else if (tempString.contains("var issue_samples"))
write = true;
}
issuesString += buffer;
} catch (IOException e) {
e.printStackTrace();
}
List<?> result = null;
ObjectMapper mapper = new ObjectMapper();
try {
Object value = mapper.readValue(issuesString, ArrayList.class);
if (value instanceof ArrayList<?>)
result = (ArrayList<?>) value;
reader.close();
} catch (JsonParseException e1) {
e1.printStackTrace();
} catch (JsonMappingException e1) {
e1.printStackTrace();
} catch (IOException e1) {
e1.printStackTrace();
}
return result;
}
// For each category, find the channel vuln and severity and pass the other work off to another method.
private List<Finding> getFindingsFromMap(List<?> map) {
if (map == null)
return null;
List<Finding> findings = new ArrayList<Finding>();
for (Object mapElement : map) {
if (mapElement instanceof HashMap<?, ?>) {
Map<?, ?> mapElementHash = (HashMap<?,?>) mapElement;
Object samples = mapElementHash.get("samples");
if (samples == null || !(samples instanceof ArrayList<?>))
continue;
ChannelSeverity cs = null;
ChannelVulnerability cv = null;
if (mapElementHash.get("severity") != null && mapElementHash.get("severity").toString() != null)
cs = getChannelSeverity(mapElementHash.get("severity").toString());
if (mapElementHash.get("type") != null && mapElementHash.get("type").toString() != null)
cv = getChannelVulnerability(mapElementHash.get("type").toString());
List<Finding> tempList = getFindingsForSingleVuln(cs, cv,
(ArrayList<?>) samples);
if (tempList != null && tempList.size() != 0)
findings.addAll(tempList);
}
}
return findings;
}
// For each channel vuln and severity, parse each path / parameter combination into a finding.
private List<Finding> getFindingsForSingleVuln(ChannelSeverity channelSeverity,
ChannelVulnerability channelVulnerability, List<?> samples) {
if (samples == null || samples.size() == 0)
return null;
List<Finding> returnList = new ArrayList<Finding>();
for (Object sample : samples) {
if (sample == null || !(sample instanceof LinkedHashMap))
continue;
Map<?, ?> findingMap = (HashMap<?, ?>) sample;
Finding finding = new Finding();
finding.setIsStatic(false);
finding.setChannelSeverity(channelSeverity);
if (channelVulnerability != null && channelVulnerability.getCode() != null && channelVulnerability.getCode().equals(INTERESTING_FILE_CODE)) {
Object extra = findingMap.get("extra");
if (extra != null && extra instanceof String &&
((String) extra).equals(DIRECTORY_LISTING)) {
ChannelVulnerability temp = getChannelVulnerability(INTERESTING_FILE_CODE + " " + DIRECTORY_LISTING);
if (temp != null)
finding.setChannelVulnerability(temp);
}
}
if (finding.getChannelVulnerability() == null)
finding.setChannelVulnerability(channelVulnerability);
finding.setSurfaceLocation(new SurfaceLocation());
String path = null, param = null, channelVulnName = null;
Object url = findingMap.get("url");
if (url != null && url instanceof String) {
Object extraObject = findingMap.get("extra");
if (extraObject != null && extraObject instanceof String) {
if (((String) extraObject).startsWith("response suggests arithmetic evaluation on server side")) {
if (((String) url).contains("-") && ((String) url).contains("?"))
param = getRegexResult((String) url, REGEX_START + "-");
}
}
if (((String) url).contains("?")) {
for (int index = 0; index < SKIPFISH_PAYLOADS.length; index ++) {
// If it has the payload, find the correct parameter and save it.
if (param == null && ((String) url).contains(SKIPFISH_PAYLOADS[index]))
param = getRegexResult((String) url, REGEX_START + SKIPFISH_PAYLOAD_REGEXES[index]);
}
path = ((String) url).substring(0, ((String) url).indexOf('?'));
} else if (zipFile != null && param == null)
param = attemptToParseParamFromHTMLRequest(findingMap);
if (path == null)
path = (String) url;
for (int index = 0; index < SKIPFISH_PAYLOADS.length; index ++)
if (path.contains(SKIPFISH_PAYLOADS[index]))
path = path.substring(0, path.indexOf(SKIPFISH_PAYLOADS[index]));
finding.getSurfaceLocation().setParameter(param);
Object requestLocation = findingMap.get("dir");
String host = null;
if (requestLocation != null && requestLocation.getClass().equals(String.class))
host = attemptToParseHostFromHTMLRequest((String) requestLocation);
if (host != null && path.contains(host)) {
finding.getSurfaceLocation().setHost(host);
finding.getSurfaceLocation().setPath(path.substring(path.indexOf(host) + host.length()));
} else {
finding.getSurfaceLocation().setPath(path);
}
finding.getSurfaceLocation().setHost(host);
}
if (channelVulnerability != null && channelVulnerability.getName() != null)
channelVulnName = channelVulnerability.getName();
finding.setNativeId(hashFindingInfo(channelVulnName, path, param));
returnList.add(finding);
}
return returnList;
}
// This is the method that tries to grab the parameter name out of the request.dat file.
// It only works if the parameter is on the bottom line in a list, which it sometimes is.
// Most parameters should be parsed before this method.
private String attemptToParseParamFromHTMLRequest(Map<?, ?> findingMap) {
if (findingMap == null || zipFile == null)
return null;
// First we need to get the file from the correct directory.
InputStream requestInputStream = null;
Object dir = findingMap.get("dir");
if (dir != null && (dir instanceof String)) {
if (folderName != null)
requestInputStream = getFileFromZip(folderName + "/" + dir.toString() + "/request.dat");
else
requestInputStream = getFileFromZip(dir.toString() + "/request.dat");
if (date == null)
attemptToParseDate(dir.toString());
}
if (requestInputStream == null)
return null;
// Then we need to grab the last line with text and look for the XSS vuln code in it (-->">'>'"<)
// It might be good to replace this with a regular expression but it would get complicated and this works.
String requestString = getStringFromInputStream(requestInputStream);
try {
requestInputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
if (requestString == null)
return null;
if (requestString.contains("\n"))
requestString = requestString.substring(requestString.lastIndexOf('\n') + 1);
if (requestString.trim().equals(""))
return null;
boolean parseFlag = false;
for (String payload : SKIPFISH_PAYLOADS) {
if (requestString.contains(payload)) {
requestString = requestString.substring(0, requestString.indexOf(payload));
parseFlag = true;
break;
}
}
Object extraObject = findingMap.get("extra");
if (extraObject != null && extraObject instanceof String) {
if (((String) extraObject).startsWith("response suggests arithmetic evaluation on server side")) {
requestString = requestString.substring(0, requestString.indexOf("-"));
parseFlag = true;
}
}
if (parseFlag && requestString.contains("=")) {
requestString = requestString.substring(0, requestString.lastIndexOf('='));
if (requestString.contains("&"))
return requestString.substring(requestString.lastIndexOf('&') + 1);
else
return requestString;
}
return null;
}
private Calendar attemptToParseDate(String responseDataAddress) {
if (zipFile == null)
return null;
InputStream requestInputStream = getFileFromZip(folderName + "/" + responseDataAddress + "/response.dat");
if (requestInputStream == null)
return null;
String responseString = getStringFromInputStream(requestInputStream);
if (responseString == null)
return null;
try {
requestInputStream.close();
} catch (IOException e) {
log.warn("Closing an inputStream failed in attemptToParseDate() in SkipfishChannelImporter.", e);
}
date = attemptToParseDateFromHTTPResponse(responseString);
return date;
}
private String attemptToParseHostFromHTMLRequest(String requestDataAddress) {
if (zipFile == null)
return null;
// First we need to get the file from the correct directory.
InputStream requestInputStream = getFileFromZip(folderName + "/" + requestDataAddress + "/request.dat");
if (requestInputStream == null) return null;
String requestString = getStringFromInputStream(requestInputStream);
if (requestString == null) return null;
try {
requestInputStream.close();
} catch (IOException e) {
log.warn("Closing an inputStream failed in attemptToParseHostFromHTMLRequest() in SkipfishChannelImporter.", e);
}
return getRegexResult(requestString, "Host: ([^\\n\\r]+)");
}
private String getStringFromInputStream(InputStream stream) {
Writer writer = new StringWriter();
String returnValue = null;
try {
IOUtils.copy(stream, writer, "UTF-8");
returnValue = writer.toString();
writer.close();
} catch (IOException e) {
e.printStackTrace();
} finally {
closeInputStream(stream);
}
return returnValue;
}
// This method looks to see if the zip file contains the folder containing everything,
// and returns the name of the folder so that paths can be correctly constructed.
private String findFolderName(ZipFile zipFile) {
if (zipFile == null)
return null;
if (zipFile.entries() != null && zipFile.entries().hasMoreElements()) {
String possibleMatch = zipFile.entries().nextElement().toString();
if (possibleMatch.charAt(0) != '_' && possibleMatch.contains("/"))
return possibleMatch.substring(0, possibleMatch.indexOf("/"));
}
return null;
}
@Override
public ScanCheckResultBean checkFile() {
ScanImportStatus returnValue = null;
InputStream sampleFileInputStream = getSampleFileInputStream();
if (sampleFileInputStream == null)
return new ScanCheckResultBean(ScanImportStatus.WRONG_FORMAT_ERROR);
List<?> map = getArrayFromSamplesFile(sampleFileInputStream);
if (map == null)
returnValue = ScanImportStatus.WRONG_FORMAT_ERROR;
if (returnValue == null && map.size() == 0)
returnValue = ScanImportStatus.EMPTY_SCAN_ERROR;
if (returnValue == null) {
checkMap(map);
if (testDate != null)
returnValue = checkTestDate();
}
try {
sampleFileInputStream.close();
} catch (IOException e) {
log.warn("Closing an inputStream failed in checkFile() in SkipfishChannelImporter.", e);
}
if (returnValue == null)
returnValue = ScanImportStatus.SUCCESSFUL_SCAN;
deleteZipFile();
return new ScanCheckResultBean(returnValue, testDate);
}
// For each category, find the channel vuln and severity and pass the other work off to another method.
private void checkMap(List<?> map) {
if (map == null)
return;
for (Object mapElement : map) {
if (mapElement == null || !(mapElement instanceof HashMap<?, ?>))
continue;
Map<?, ?> mapElementHash = (HashMap<?,?>) mapElement;
Object samples = mapElementHash.get("samples");
if (samples == null || !(samples instanceof ArrayList<?>))
continue;
for (Object sample : (ArrayList<?>) samples) {
if (sample == null || !(sample instanceof LinkedHashMap))
continue;
Map<?, ?> findingMap = (HashMap<?, ?>) sample;
Object dir = findingMap.get("dir");
if (dir != null && (dir instanceof String) && testDate == null) {
testDate = attemptToParseDate(dir.toString());
if (testDate != null)
return;
}
}
}
}
}