/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nutch.tika;
import org.apache.nutch.parse.HTMLMetaTags;
import org.apache.nutch.parse.tika.HTMLMetaProcessor;
import java.io.ByteArrayInputStream;
import java.net.URL;
import org.xml.sax.*;
import org.w3c.dom.*;
import org.apache.html.dom.*;
import org.cyberneko.html.parsers.DOMFragmentParser;
import org.junit.Assert;
import org.junit.Test;
/** Unit tests for HTMLMetaProcessor. */
public class TestRobotsMetaProcessor {
/*
*
* some sample tags:
*
* <meta name="robots" content="index,follow"> <meta name="robots"
* content="noindex,follow"> <meta name="robots" content="index,nofollow">
* <meta name="robots" content="noindex,nofollow">
*
* <META HTTP-EQUIV="Pragma" CONTENT="no-cache">
*/
public static String[] tests = {
"<html><head><title>test page</title>"
+ "<META NAME=\"ROBOTS\" CONTENT=\"NONE\"> "
+ "<META HTTP-EQUIV=\"PRAGMA\" CONTENT=\"NO-CACHE\"> "
+ "</head><body>" + " some text" + "</body></html>",
"<html><head><title>test page</title>"
+ "<meta name=\"robots\" content=\"all\"> "
+ "<meta http-equiv=\"pragma\" content=\"no-cache\"> "
+ "</head><body>" + " some text" + "</body></html>",
"<html><head><title>test page</title>"
+ "<MeTa NaMe=\"RoBoTs\" CoNtEnT=\"nOnE\"> "
+ "<MeTa HtTp-EqUiV=\"pRaGmA\" cOnTeNt=\"No-CaChE\"> "
+ "</head><body>" + " some text" + "</body></html>",
"<html><head><title>test page</title>"
+ "<meta name=\"robots\" content=\"none\"> " + "</head><body>"
+ " some text" + "</body></html>",
"<html><head><title>test page</title>"
+ "<meta name=\"robots\" content=\"noindex,nofollow\"> "
+ "</head><body>" + " some text" + "</body></html>",
"<html><head><title>test page</title>"
+ "<meta name=\"robots\" content=\"noindex,follow\"> "
+ "</head><body>" + " some text" + "</body></html>",
"<html><head><title>test page</title>"
+ "<meta name=\"robots\" content=\"index,nofollow\"> "
+ "</head><body>" + " some text" + "</body></html>",
"<html><head><title>test page</title>"
+ "<meta name=\"robots\" content=\"index,follow\"> "
+ "<base href=\"http://www.nutch.org/\">" + "</head><body>"
+ " some text" + "</body></html>",
"<html><head><title>test page</title>" + "<meta name=\"robots\"> "
+ "<base href=\"http://www.nutch.org/base/\">" + "</head><body>"
+ " some text" + "</body></html>",
};
public static final boolean[][] answers = { { true, true, true }, // NONE
{ false, false, true }, // all
{ true, true, true }, // nOnE
{ true, true, false }, // none
{ true, true, false }, // noindex,nofollow
{ true, false, false }, // noindex,follow
{ false, true, false }, // index,nofollow
{ false, false, false }, // index,follow
{ false, false, false }, // missing!
};
private URL[][] currURLsAndAnswers;
@Test
public void testRobotsMetaProcessor() {
DOMFragmentParser parser = new DOMFragmentParser();
;
try {
currURLsAndAnswers = new URL[][] {
{ new URL("http://www.nutch.org"), null },
{ new URL("http://www.nutch.org"), null },
{ new URL("http://www.nutch.org"), null },
{ new URL("http://www.nutch.org"), null },
{ new URL("http://www.nutch.org"), null },
{ new URL("http://www.nutch.org"), null },
{ new URL("http://www.nutch.org"), null },
{ new URL("http://www.nutch.org/foo/"),
new URL("http://www.nutch.org/") },
{ new URL("http://www.nutch.org"),
new URL("http://www.nutch.org/base/") } };
} catch (Exception e) {
Assert.assertTrue("couldn't make test URLs!", false);
}
for (int i = 0; i < tests.length; i++) {
byte[] bytes = tests[i].getBytes();
DocumentFragment node = new HTMLDocumentImpl().createDocumentFragment();
try {
parser.parse(new InputSource(new ByteArrayInputStream(bytes)), node);
} catch (Exception e) {
e.printStackTrace();
}
HTMLMetaTags robotsMeta = new HTMLMetaTags();
HTMLMetaProcessor.getMetaTags(robotsMeta, node, currURLsAndAnswers[i][0]);
Assert.assertTrue("got index wrong on test " + i,
robotsMeta.getNoIndex() == answers[i][0]);
Assert.assertTrue("got follow wrong on test " + i,
robotsMeta.getNoFollow() == answers[i][1]);
Assert.assertTrue("got cache wrong on test " + i,
robotsMeta.getNoCache() == answers[i][2]);
Assert
.assertTrue(
"got base href wrong on test " + i + " (got "
+ robotsMeta.getBaseHref() + ")",
((robotsMeta.getBaseHref() == null) && (currURLsAndAnswers[i][1] == null))
|| ((robotsMeta.getBaseHref() != null) && robotsMeta
.getBaseHref().equals(currURLsAndAnswers[i][1])));
}
}
}