/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.zaproxy.zap.spider.parser;
import java.util.Locale;
import java.util.StringTokenizer;
import net.htmlparser.jericho.Source;
import org.parosproxy.paros.Constant;
import org.parosproxy.paros.network.HttpMessage;
import org.zaproxy.zap.spider.SpiderParam;
/**
* The Class SpiderRobotstxtParser used for parsing Robots.txt files.
*/
public class SpiderRobotstxtParser extends SpiderParser {
private static final String PATTERNS_USERAGENT = "(?i)^User-agent:.*";
private static final String PATTERNS_DISALLOW = "(?i)Disallow:.*";
private static final String PATTERNS_ALLOW = "(?i)Allow:.*";
private static final int PATTERNS_USERAGENT_LENGTH = 11;
private static final int PATTERNS_DISALLOW_LENGTH = 9;
private static final int PATTERNS_ALLOW_LENGTH = 6;
/** The params. */
private SpiderParam params;
/**
* Instantiates a new spider robotstxt parser.
*
* @param params the params
*/
public SpiderRobotstxtParser(SpiderParam params) {
super();
this.params = params;
}
@Override
public boolean parseResource(HttpMessage message, Source source, int depth) {
if (message == null || !params.isParseRobotsTxt()) {
return false;
}
log.debug("Parsing a robots.txt resource...");
// Get the response content
String content = message.getResponseBody().toString();
// Get the context (base url)
String baseURL;
baseURL = message.getRequestHeader().getURI().toString();
@SuppressWarnings("unused")
// for now...
boolean inMatchingUserAgent = false;
// Parse each line in the Spider.txt file
StringTokenizer st = new StringTokenizer(content, "\n");
while (st.hasMoreTokens()) {
String line = st.nextToken();
// Remove comments
int commentStart = line.indexOf("#");
if (commentStart != -1) {
line = line.substring(0, commentStart);
}
// remove HTML markup and clean
line = line.replaceAll("<[^>]+>", "");
line = line.trim();
// If nothing's left, skip
if (line.length() == 0) {
continue;
}
log.debug("Processing robots.txt line: " + line);
// If the line is for defining the user agent
if (line.matches(PATTERNS_USERAGENT)) {
String ua = line.substring(PATTERNS_USERAGENT_LENGTH).trim().toLowerCase(Locale.ENGLISH);
if (ua.equals("*") || ua.contains(Constant.USER_AGENT)) {
log.debug("Parsing robots.txt file. Starting section applying to spider.");
inMatchingUserAgent = true;
} else {
log.debug("Parsing robots.txt file. Start section not applying to spider.");
inMatchingUserAgent = false;
}
// If the line is for defining a DISALLOW pattern
} else if (line.matches(PATTERNS_DISALLOW)) {
// The spider should explore URIs no matter who the pattern applies to
// if (!inMatchingUserAgent) {
// continue;
// }
String path = line.substring(PATTERNS_DISALLOW_LENGTH).trim();
// Clean the path
if (path.endsWith("*")) {
path = path.substring(0, path.length() - 1);
}
path = path.trim();
// Submit the found url
if (path.length() > 0) {
processURL(message, depth, path, baseURL);
}
// If the line is for defining an ALLOW pattern
} else if (line.matches(PATTERNS_ALLOW)) {
// The spider should explore URIs no matter who the pattern applies to
// if (!inMatchingUserAgent) {
// continue;
// }
// Get the cleaned path
String path = line.substring(PATTERNS_ALLOW_LENGTH).trim();
if (path.endsWith("*")) {
path = path.substring(0, path.length() - 1);
}
path = path.trim();
// Submit the found url
if (path.length() > 0) {
processURL(message, depth, path, baseURL);
}
}
}
// We consider the message fully parsed, so it doesn't get parsed by 'fallback' parsers
return true;
}
@Override
public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyParsed) {
// If it's a robots.txt file
return path != null && path.equalsIgnoreCase("/robots.txt");
}
}