/*
* Tanaguru - Automated webpage assessment
* Copyright (C) 2008-2015 Tanaguru.org
*
* This file is part of Tanaguru.
*
* Tanaguru is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* Contact us by mail: tanaguru AT tanaguru DOT org
*/
package org.tanaguru.processing;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.*;
import javax.annotation.Nullable;
import javax.xml.xpath.*;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.tanaguru.entity.audit.*;
import org.tanaguru.entity.service.audit.EvidenceDataService;
import org.tanaguru.entity.service.audit.EvidenceElementDataService;
import org.tanaguru.entity.service.audit.ProcessRemarkDataService;
import org.tanaguru.service.ProcessRemarkService;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
/**
*
* @author jkowalczyk
*/
public class ProcessRemarkServiceImpl implements ProcessRemarkService {
private static final Logger LOGGER = Logger.getLogger(ProcessRemarkServiceImpl.class);
// private static final String ESCAPED_DOUBLE_QUOTE = """;
private static final String ESCAPED_OPEN_TAG = "<";
private static final String ESCAPED_CLOSE_TAG = ">";
private static final String CLOSURE_TAG_OCCUR = "/";
private static final String AUTO_CLOSE_TAG_OCCUR =
CLOSURE_TAG_OCCUR+ESCAPED_CLOSE_TAG;
private static final String CLOSE_TAG_OCCUR =
ESCAPED_OPEN_TAG + CLOSURE_TAG_OCCUR;
private static final String CSS_SELECTOR_EVIDENCE = "Css-Selector";
private static final String CSS_FILENAME_EVIDENCE = "Css-Filename";
private static final String START_COMMENT_OCCURENCE = "<!--";
private static final String END_COMMENT_OCCURENCE = "-->";
private final XPath xpath = XPathFactory.newInstance().newXPath();
private Document document;
private org.jsoup.nodes.Document jsoupDocument;
/**
*
* @param processRemarkDataService
* @param evidenceElementDataService
* @param evidenceDataService
*/
public ProcessRemarkServiceImpl(
ProcessRemarkDataService processRemarkDataService,
EvidenceElementDataService evidenceElementDataService,
EvidenceDataService evidenceDataService) {
super();
this.processRemarkDataService = processRemarkDataService;
this.evidenceElementDataService = evidenceElementDataService;
this.evidenceDataService = evidenceDataService;
}
/**
*
* @param document
*/
public void setDocument(Document document) {
this.document = document;
}
public void setJsoupDocument(org.jsoup.nodes.Document document) {
this.jsoupDocument = document;
}
private Collection<ProcessRemark> remarkSet;
@Override
public Collection<ProcessRemark> getRemarkList() {
return this.remarkSet;
}
private List<String> evidenceElementList = new ArrayList<>();
@Override
public void addEvidenceElement(String element) {
if (!evidenceElementList.contains(element)) {
evidenceElementList.add(element);
}
}
@Override
public void setEvidenceElementList(Collection<String> element) {
evidenceElementList.addAll(element);
}
private ProcessRemarkDataService processRemarkDataService;
public ProcessRemarkDataService getProcessRemarkDataService() {
return processRemarkDataService;
}
public void setProcessRemarkFactory(ProcessRemarkDataService processRemarkDataService) {
this.processRemarkDataService = processRemarkDataService;
}
private EvidenceElementDataService evidenceElementDataService;
@Override
public EvidenceElementDataService getEvidenceElementDataService() {
return evidenceElementDataService;
}
public void setEvidenceElementFactory(EvidenceElementDataService evidenceElementDataService) {
this.evidenceElementDataService = evidenceElementDataService;
}
private EvidenceDataService evidenceDataService;
@Override
public EvidenceDataService getEvidenceDataService() {
return evidenceDataService;
}
public void setEvidenceDataService(EvidenceDataService evidenceDataService) {
this.evidenceDataService = evidenceDataService;
}
private Map<Integer, String> sourceCodeWithLine = null;
private Map<Integer, String> rawSourceCodeWithLine = null;
/**
* Local map of evidence to avoid multiple access to database
*/
private final Map<String, Evidence> evidenceMap = new HashMap<>();
@Override
public void initializeService(Document document, String adaptedContent) {
if (document != null) {
this.document = document;
}
if (adaptedContent != null && sourceCodeWithLine == null) {
initializeSourceCodeMap(adaptedContent);
}
// call the reset service to instanciated local remarks collection
// and evidence elements collection
resetService();
}
/**
* The initialisation of the jquery like should be called once for each
* tested SSP. It stores the current document and initialised a map with the
* source where each key is the line number. This map is then used to
* retrieve the line number of an element.
*
* @param document
* @param adaptedContent
*/
@Override
public void initializeService(org.jsoup.nodes.Document document, String adaptedContent) {
Date beginDate = null;
if (document != null) {
this.jsoupDocument = document;
}
if (LOGGER.isDebugEnabled()) {
beginDate = new Date();
LOGGER.debug("Initialising source Map by line");
}
if (adaptedContent != null) {
initializeRawSourceCodeMap(adaptedContent);
}
if (LOGGER.isDebugEnabled()) {
Date endDate = new Date();
LOGGER.debug("initialisation of source map by line took "
+ (endDate.getTime()-beginDate.getTime()) +"ms");
}
// call the reset service to instanciated local remarks collection
// and evidence elements collection
resetService();
}
@Override
public void resetService() {
LOGGER.debug("Service is reset");
remarkSet = new LinkedHashSet<>();
evidenceElementList = new LinkedList<>();
}
@Override
public ProcessRemark createProcessRemark(
TestSolution processResult,
String messageCode) {
return processRemarkDataService.getProcessRemark(processResult, messageCode);
}
@Override
public void addProcessRemark(TestSolution processResult, String messageCode) {
remarkSet.add(processRemarkDataService.getProcessRemark(processResult, messageCode));
}
@Override
public void addSourceCodeRemark(TestSolution processResult, Node node,
String messageCode, String attributeName) {
remarkSet.add(createSourceCodeRemark(
processResult,
node,
messageCode,
attributeName));
}
@Override
public void addSourceCodeRemarkOnElement(TestSolution processResult, Element element,
String messageCode) {
remarkSet.add(createSourceCodeRemark(
processResult,
element,
messageCode));
}
@Override
public void addSourceCodeRemarkOnElement(
TestSolution processResult,
Element element,
String messageCode,
Collection<EvidenceElement> evidenceElementList) {
SourceCodeRemark remark =
processRemarkDataService.getSourceCodeRemark(
processResult,
messageCode);
if (element != null) {
remark.setLineNumber(searchElementLineNumber(element));
remark.setTarget(element.nodeName());
remark.setSnippet(getSnippetFromElement(element));
} else {
remark.setLineNumber(-1);
}
if (CollectionUtils.isNotEmpty(evidenceElementList)) {
for (EvidenceElement ee : evidenceElementList) {
remark.addElement(ee);
ee.setProcessRemark(remark);
}
}
remarkSet.add(remark);
}
@Override
public void addSourceCodeRemark(
TestSolution processResult,
String targetValue,
String messageCode,
EvidenceElement... evidenceElementList) {
SourceCodeRemark remark =
processRemarkDataService.getSourceCodeRemark(
targetValue,
processResult,
messageCode,
-1);
for (EvidenceElement ee : evidenceElementList) {
remark.addElement(ee);
ee.setProcessRemark(remark);
}
remarkSet.add(remark);
}
@Override
public void addSourceCodeRemark(
TestSolution processResult,
Node node,
String messageCode,
Collection<EvidenceElement> evidenceElementList) {
SourceCodeRemark remark =
processRemarkDataService.getSourceCodeRemark(
processResult,
messageCode);
if (node != null) {
remark.setLineNumber(searchNodeLineNumber(node));
} else {
remark.setLineNumber(-1);
}
for (EvidenceElement element : evidenceElementList) {
remark.addElement(element);
element.setProcessRemark(remark);
}
remarkSet.add(remark);
}
@Override
public void addCssCodeRemark(TestSolution processResult,
String selectorValue,
String messageCode,
String attrName,
String fileName) {// XXX
SourceCodeRemark remark =
processRemarkDataService.getSourceCodeRemark(
processResult,
messageCode);
// This a fake sourceCode Remark, the line number cannot be found
// we use a sourceCodeRemark here to add evidence elements
remark.setLineNumber(-1);
EvidenceElement evidenceElement =
evidenceElementDataService.getEvidenceElement(
remark,
attrName,
getEvidence(DEFAULT_EVIDENCE));
remark.addElement(evidenceElement);
try {
if (selectorValue != null) {
EvidenceElement cssSelectorEvidenceElement =
evidenceElementDataService.getEvidenceElement(
remark,
selectorValue,
getEvidence(CSS_SELECTOR_EVIDENCE));
remark.addElement(cssSelectorEvidenceElement);
}
if (fileName != null) {
EvidenceElement fileNameEvidenceElement =
evidenceElementDataService.getEvidenceElement(
remark,
fileName,
getEvidence(CSS_FILENAME_EVIDENCE));
remark.addElement(fileNameEvidenceElement);
}
} catch (ClassCastException ex) {
LOGGER.error(ex.getMessage());
}
remarkSet.add(remark);
}
@Override
public void addSourceCodeRemark(
TestSolution processResult,
Node node,
String messageCode,
String elementType,
String elementName) {
throw new UnsupportedOperationException("Not supported yet.");
}
/**
*
* @param node
* @return
*/
private int searchNodeLineNumber(Node node) {
int nodeIndex = getNodeIndex(node);
int lineNumber = 0;
boolean found = false;
boolean isWithinComment = false;
Iterator<Map.Entry<Integer, String>> iter = sourceCodeWithLine.entrySet().iterator();
String codeLine;
while (iter.hasNext() && !found) {
Map.Entry<Integer, String> entry = iter.next();
int myLineNumber = entry.getKey();
int index = 0;
while (index != -1) {
codeLine = entry.getValue().toLowerCase();
int characterPositionOri = index;
index = codeLine.indexOf("<" + node.getNodeName().toLowerCase() + ">",
index);
if (index == -1) {
index = codeLine.indexOf("<" + node.getNodeName().toLowerCase() + " ",
characterPositionOri);
}
int startCommentIndex = codeLine.indexOf(
START_COMMENT_OCCURENCE, characterPositionOri);
int endCommentIndex = codeLine.indexOf(
END_COMMENT_OCCURENCE, characterPositionOri);
if (index != -1) { // if an occurence of the tag is found
if (!isWithinComment
&& !(startCommentIndex != -1 && index > startCommentIndex)
&& !(endCommentIndex != -1 && index < endCommentIndex)) { // if a comment is not currently opened or a comment is found on the current line and the occurence is not within
if (nodeIndex == 0) {
found = true;
lineNumber = myLineNumber;
break;
}
nodeIndex--;
}
index += node.getNodeName().length();
}
// if a "start comment" occurence is found on the line,
// the boolean isWithinComment is set to true. Thus, while a
// "end comment" is not found, all the occurences of the
// wanted node will be ignored
if (!isWithinComment && startCommentIndex != -1 && endCommentIndex == -1) {
isWithinComment = true;
} else if (isWithinComment && endCommentIndex != -1 && startCommentIndex < endCommentIndex) {
isWithinComment = false;
}
}
}
return lineNumber;
}
/**
*
* @param node
* @return
*/
private int searchElementLineNumber(Element element) {
int nodeIndex = getElementIndex(element);
int lineNumber = 0;
boolean found = false;
boolean isWithinComment = false;
Iterator<Map.Entry<Integer, String>> iter = rawSourceCodeWithLine.entrySet().iterator();
String codeLine;
while (iter.hasNext() && !found) {
Map.Entry<Integer, String> entry = iter.next();
int myLineNumber = entry.getKey();
int index = 0;
while (index != -1) {
codeLine = entry.getValue().toLowerCase();
int characterPositionOri = index;
index = codeLine.indexOf("<" + element.nodeName() + ">",
index);
if (index == -1) {
index = codeLine.indexOf("<" + element.nodeName() + " ",
characterPositionOri);
}
int startCommentIndex = codeLine.indexOf(
START_COMMENT_OCCURENCE, characterPositionOri);
int endCommentIndex = codeLine.indexOf(
END_COMMENT_OCCURENCE, characterPositionOri);
if (index != -1) { // if an occurence of the tag is found
if (!isWithinComment
&& !(startCommentIndex != -1 && index > startCommentIndex)
&& !(endCommentIndex != -1 && index < endCommentIndex)) { // if a comment is not currently opened or a comment is found on the current line and the occurence is not within
if (nodeIndex == 0) {
found = true;
lineNumber = myLineNumber;
break;
}
nodeIndex--;
}
index += element.nodeName().length();
}
// if a "start comment" occurence is found on the line,
// the boolean isWithinComment is set to true. Thus, while a
// "end comment" is not found, all the occurences of the
// wanted node will be ignored
if (!isWithinComment && startCommentIndex != -1 && endCommentIndex == -1) {
isWithinComment = true;
} else if (isWithinComment && endCommentIndex != -1 && startCommentIndex < endCommentIndex) {
isWithinComment = false;
}
}
}
return lineNumber;
}
/**
* This methods search the line where the current node is present in
* the source code
* @param node
* @return
*/
private int getNodeIndex(Node node) {
try {
XPathExpression xPathExpression = xpath.compile("//" + node.getNodeName().toUpperCase());
Object result = xPathExpression.evaluate(document,
XPathConstants.NODESET);
NodeList nodeList = (NodeList) result;
for (int i = 0; i < nodeList.getLength(); i++) {
Node current = nodeList.item(i);
if (current.equals(node)) {
return i;
}
}
} catch (XPathExpressionException ex) {
LOGGER.error("Error occured while searching index of a node "+
ex.getMessage());
throw new RuntimeException(ex);
}
return -1;
}
/**
* This methods search the line where the current node is present in
* the source code
* @param node
* @return
*/
private int getElementIndex(Element element) {
Elements elements = jsoupDocument.getElementsByTag(element.tagName());
for (int i = 0; i < elements.size(); i++) {
Element current = elements.get(i);
if (current.equals(element)) {
return i;
}
}
return -1;
}
/**
* Initialisation of a local map that handles each source code line,
* keyed by the line number
*
* @param adaptedContent
*/
private void initializeSourceCodeMap(String adaptedContent) {
sourceCodeWithLine = new LinkedHashMap<>();
int lineNumber = 1;
StringReader sr = new StringReader(adaptedContent);
BufferedReader br = new BufferedReader(sr);
String line;
try {
while ((line = br.readLine()) != null) {
sourceCodeWithLine.put(lineNumber, line);
lineNumber++;
}
} catch (IOException ex) {
LOGGER.error("Error occured while initialize source code map "+
ex.getMessage());
}
}
/**
* Initialisation of a local map that handles each source code line,
* keyed by the line number
*
* @param adaptedContent
*/
private void initializeRawSourceCodeMap(String rawSource) {
rawSourceCodeWithLine = new LinkedHashMap<>();
int lineNumber = 1;
StringReader sr = new StringReader(rawSource);
BufferedReader br = new BufferedReader(sr);
String line;
try {
while ((line = br.readLine()) != null) {
if (StringUtils.isNotBlank(line)) {
rawSourceCodeWithLine.put(lineNumber, line);
lineNumber++;
}
}
} catch (IOException ex) {
LOGGER.error("Error occured while initialize raw source code map "+
ex.getMessage());
}
}
@Override
public EvidenceElement getEvidenceElement(String evidenceCode, String evidenceValue) {
return evidenceElementDataService.getEvidenceElement(
StringUtils.trim(evidenceValue),
getEvidence(evidenceCode));
}
@Override
public SourceCodeRemark createSourceCodeRemark(
TestSolution processResult,
Node node,
String messageCode,
String elementName) {
SourceCodeRemark remark =
processRemarkDataService.getSourceCodeRemark(
processResult,
messageCode);
remark.setLineNumber(searchNodeLineNumber(node));
EvidenceElement evidenceElement =
evidenceElementDataService.getEvidenceElement(
remark,
elementName,
getEvidence(DEFAULT_EVIDENCE));
for (String attr : evidenceElementList) {
if (node.getAttributes().getNamedItem(attr) != null) {
EvidenceElement evidenceElementSup =
evidenceElementDataService.getEvidenceElement(
remark,
node.getAttributes().getNamedItem(attr).getNodeValue(),
getEvidence(attr));
remark.addElement(evidenceElementSup);
}
}
remark.addElement(evidenceElement);
return remark;
}
@Override
public SourceCodeRemark createSourceCodeRemark(
TestSolution processResult,
Element element,
String messageCode) {
SourceCodeRemark remark =
processRemarkDataService.getSourceCodeRemark(
element.nodeName(),
processResult,
messageCode,
searchElementLineNumber(element));
remark.setSnippet(getSnippetFromElement(element));
for (String attr : evidenceElementList) {
EvidenceElement evidenceElementSup;
if (StringUtils.equalsIgnoreCase(attr, "text")) {
evidenceElementSup = getEvidenceElement(attr, element.text());
} else {
evidenceElementSup = getEvidenceElement(attr, element.attr(attr));
}
remark.addElement(evidenceElementSup);
}
return remark;
}
@Override
public void addConsolidationRemark(
TestSolution processResult,
String messageCode,
String value,
String url) {
remarkSet.add(createConsolidationRemark(
processResult,
messageCode,
value,
url));
}
@Override
public void addConsolidationRemark(
TestSolution processResult,
String messageCode) {
remarkSet.add(createConsolidationRemark(
processResult,
messageCode));
}
@Override
public void addProcessRemark(
TestSolution processResult,
String messageCode,
Collection<EvidenceElement> evidenceElementList) {
remarkSet.add(createProcessRemark(
processResult,
messageCode,
evidenceElementList));
}
@Override
public ProcessRemark createProcessRemark(
TestSolution processResult,
String messageCode,
Collection<EvidenceElement> evidenceElementList) {
ProcessRemark remark = processRemarkDataService.getProcessRemark(processResult,messageCode);
for (EvidenceElement element : evidenceElementList) {
remark.addElement(element);
element.setProcessRemark(remark);
}
return remark;
}
@Override
public ProcessRemark createConsolidationRemark(
TestSolution processResult,
String messageCode) {
return processRemarkDataService.getProcessRemark(processResult, messageCode);
}
@Override
public ProcessRemark createConsolidationRemark(
TestSolution processResult,
String messageCode,
@Nullable String value,
@Nullable String url) {
ProcessRemark remark = createConsolidationRemark(processResult, messageCode);
if (value != null) {
EvidenceElement evidenceElement =
evidenceElementDataService.getEvidenceElement(
remark,
value,
getEvidence(DEFAULT_EVIDENCE));
remark.addElement(evidenceElement);
}
if (url != null) {
EvidenceElement evidenceElement =
evidenceElementDataService.getEvidenceElement(
remark,
url,
getEvidence(URL_EVIDENCE));
remark.addElement(evidenceElement);
}
return remark;
}
/**
* Return an evidence instance for a given code. This method avoids multiple
* access to mysql databases, by maintaining a map.
* @param code
* @return
*/
public Evidence getEvidence(String code) {
if (evidenceMap.containsKey(code)) {
return evidenceMap.get(code);
} else {
Evidence evidence = evidenceDataService.findByCode(code);
evidenceMap.put(code, evidence);
return evidence;
}
}
/**
*
* @param element
* @return
*/
public String getSnippetFromElement(Element element) {
String elementHtml = StringEscapeUtils.escapeHtml4(
StringUtil.normaliseWhitespace(element.outerHtml())).trim();
if (element.children().isEmpty() || elementHtml.length() <= SNIPPET_MAX_LENGTH) {
return elementHtml;
}
return properlyCloseSnippet(
element,
elementHtml,
elementHtml.substring(0, SNIPPET_MAX_LENGTH));
}
/**
*
* @param element
* @param originalElementHtml
* @param truncatedElementHtml
* @return
*/
private String properlyCloseSnippet (
Element element,
String originalElementHtml,
String truncatedElementHtml) {
if (isElementAutoClose(originalElementHtml)) {
return originalElementHtml;
}
if (getElementCurrentlyOpenCount(truncatedElementHtml) == 1) {
return closeInnerElement(originalElementHtml, truncatedElementHtml);
} else if (getElementCurrentlyOpenCount(truncatedElementHtml) > 1) {
truncatedElementHtml =
closeInnerElement(originalElementHtml, truncatedElementHtml);
return closeElement(truncatedElementHtml, element.tagName());
} else {
return closeElement(truncatedElementHtml, element.tagName());
}
}
/**
*
* @param elementHtml
* @return
*/
private boolean isElementAutoClose(String elementHtml) {
return StringUtils.endsWith(elementHtml, AUTO_CLOSE_TAG_OCCUR);
}
/**
*
* @param elementHtml
* @return
*/
private int getElementCurrentlyOpenCount(String elementHtml) {
int openTagCounter =
StringUtils.countMatches(elementHtml, ESCAPED_OPEN_TAG)
-
StringUtils.countMatches(elementHtml, CLOSE_TAG_OCCUR);
int closureTagCounter =
StringUtils.countMatches(elementHtml, AUTO_CLOSE_TAG_OCCUR)
+
StringUtils.countMatches(elementHtml, CLOSE_TAG_OCCUR) ;
return (openTagCounter - closureTagCounter);
}
/**
*
* @param originalElementHtml
* @return
*/
private String closeElement(String elementHtml, String elementName) {
StringBuilder strb = new StringBuilder();
strb.append(elementHtml);
strb.append("[...]");
strb.append(ESCAPED_OPEN_TAG);
strb.append(CLOSURE_TAG_OCCUR);
strb.append(elementName);
strb.append(ESCAPED_CLOSE_TAG);
return strb.toString();
}
/**
*
* @param originalElementHtml
* @param truncatedElementHtml
* @param indexOfElementToClose
* @return
*/
private String closeInnerElement (
String originalElementHtml,
String truncatedElementHtml) {
int startPos = truncatedElementHtml.length();
int indexOfElementCloser =
StringUtils.indexOf(originalElementHtml, ESCAPED_CLOSE_TAG, startPos);
int indexOfElementAutoCloser =
StringUtils.indexOf(originalElementHtml, AUTO_CLOSE_TAG_OCCUR, startPos);
String innerClosedElement = StringUtils.substring(
originalElementHtml,
0,
(indexOfElementCloser+ESCAPED_CLOSE_TAG.length()));
// if the current element is auto-close, return current well-closed element
if (indexOfElementAutoCloser == (indexOfElementCloser-1) ) {
return innerClosedElement;
}
// if the current element is not auto-close, get the name of it and
// and properly close it
int indexOfElementOpenTagOpener =
StringUtils.lastIndexOf(originalElementHtml, ESCAPED_OPEN_TAG, startPos);
int indexOfElementOpenTagClose =
StringUtils.indexOf(originalElementHtml, ' ', indexOfElementOpenTagOpener);
String elementName =
StringUtils.substring(
originalElementHtml,
indexOfElementOpenTagOpener + ESCAPED_OPEN_TAG.length(),
indexOfElementOpenTagClose);
return closeElement(innerClosedElement, elementName);
}
}