package org.apache.tika.parser;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static org.apache.tika.TikaTest.assertContains;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.InputStream;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaMetadataKeys;
import org.apache.tika.parser.utils.CommonsDigester;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.ContentHandlerFactory;
import org.junit.Test;
import org.xml.sax.helpers.DefaultHandler;
public class RecursiveParserWrapperTest {
@Test
public void testBasicXML() throws Exception {
List<Metadata> list = getMetadata(new Metadata(),
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.XML, -1));
Metadata container = list.get(0);
String content = container.get(RecursiveParserWrapper.TIKA_CONTENT);
//not much differentiates html from xml in this test file
assertTrue(content.indexOf("<p class=\"header\" />") > -1);
}
@Test
public void testBasicHTML() throws Exception {
List<Metadata> list = getMetadata(new Metadata(),
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.HTML, -1));
Metadata container = list.get(0);
String content = container.get(RecursiveParserWrapper.TIKA_CONTENT);
//not much differentiates html from xml in this test file
assertTrue(content.indexOf("<p class=\"header\"></p>") > -1);
}
@Test
public void testBasicText() throws Exception {
List<Metadata> list = getMetadata(new Metadata(),
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1));
Metadata container = list.get(0);
String content = container.get(RecursiveParserWrapper.TIKA_CONTENT);
assertTrue(content.indexOf("<p ") < 0);
assertTrue(content.indexOf("embed_0") > -1);
}
@Test
public void testIgnoreContent() throws Exception {
List<Metadata> list = getMetadata(new Metadata(),
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.IGNORE, -1));
Metadata container = list.get(0);
String content = container.get(RecursiveParserWrapper.TIKA_CONTENT);
assertNull(content);
}
@Test
public void testCharLimit() throws Exception {
ParseContext context = new ParseContext();
Metadata metadata = new Metadata();
Parser wrapped = new AutoDetectParser();
RecursiveParserWrapper wrapper = new RecursiveParserWrapper(wrapped,
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, 60));
InputStream stream = RecursiveParserWrapperTest.class.getResourceAsStream(
"/test-documents/test_recursive_embedded.docx");
wrapper.parse(stream, new DefaultHandler(), metadata, context);
List<Metadata> list = wrapper.getMetadata();
assertEquals(5, list.size());
int wlr = 0;
for (Metadata m : list) {
String limitReached = m.get(RecursiveParserWrapper.WRITE_LIMIT_REACHED);
if (limitReached != null && limitReached.equals("true")) {
wlr++;
}
}
assertEquals(1, wlr);
}
@Test
public void testMaxEmbedded() throws Exception {
int maxEmbedded = 4;
int totalNoLimit = 12;//including outer container file
ParseContext context = new ParseContext();
Metadata metadata = new Metadata();
String limitReached = null;
Parser wrapped = new AutoDetectParser();
RecursiveParserWrapper wrapper = new RecursiveParserWrapper(wrapped,
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1));
InputStream stream = RecursiveParserWrapperTest.class.getResourceAsStream(
"/test-documents/test_recursive_embedded.docx");
wrapper.parse(stream, new DefaultHandler(), metadata, context);
List<Metadata> list = wrapper.getMetadata();
//test default
assertEquals(totalNoLimit, list.size());
limitReached = list.get(0).get(RecursiveParserWrapper.EMBEDDED_RESOURCE_LIMIT_REACHED);
assertNull(limitReached);
wrapper.reset();
stream.close();
//test setting value
metadata = new Metadata();
stream = RecursiveParserWrapperTest.class.getResourceAsStream(
"/test-documents/test_recursive_embedded.docx");
wrapper.setMaxEmbeddedResources(maxEmbedded);
wrapper.parse(stream, new DefaultHandler(), metadata, context);
list = wrapper.getMetadata();
//add 1 for outer container file
assertEquals(maxEmbedded + 1, list.size());
limitReached = list.get(0).get(RecursiveParserWrapper.EMBEDDED_RESOURCE_LIMIT_REACHED);
assertEquals("true", limitReached);
wrapper.reset();
stream.close();
//test setting value < 0
metadata = new Metadata();
stream = RecursiveParserWrapperTest.class.getResourceAsStream(
"/test-documents/test_recursive_embedded.docx");
wrapper.setMaxEmbeddedResources(-2);
wrapper.parse(stream, new DefaultHandler(), metadata, context);
assertEquals(totalNoLimit, list.size());
limitReached = list.get(0).get(RecursiveParserWrapper.EMBEDDED_RESOURCE_LIMIT_REACHED);
assertNull(limitReached);
}
@Test
public void testEmbeddedResourcePath() throws Exception {
Set<String> targets = new HashSet<String>();
targets.add("/embed1.zip");
targets.add("/embed1.zip/embed2.zip");
targets.add("/embed1.zip/embed2.zip/embed3.zip");
targets.add("/embed1.zip/embed2.zip/embed3.zip/embed4.zip");
targets.add("/embed1.zip/embed2.zip/embed3.zip/embed4.zip/embed4.txt");
targets.add("/embed1.zip/embed2.zip/embed3.zip/embed3.txt");
targets.add("/embed1.zip/embed2.zip/embed2a.txt");
targets.add("/embed1.zip/embed2.zip/embed2b.txt");
targets.add("/embed1.zip/embed1b.txt");
targets.add("/embed1.zip/embed1a.txt");
targets.add("/image1.emf");
Metadata metadata = new Metadata();
metadata.set(Metadata.RESOURCE_NAME_KEY, "test_recursive_embedded.docx");
List<Metadata> list = getMetadata(metadata,
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.XML, -1));
Metadata container = list.get(0);
String content = container.get(RecursiveParserWrapper.TIKA_CONTENT);
assertTrue(content.indexOf("<p class=\"header\" />") > -1);
Set<String> seen = new HashSet<String>();
for (Metadata m : list) {
String path = m.get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH);
if (path != null) {
seen.add(path);
}
}
assertEquals(targets, seen);
}
@Test
public void testEmbeddedNPE() throws Exception {
Metadata metadata = new Metadata();
metadata.set(Metadata.RESOURCE_NAME_KEY, "test_recursive_embedded_npe.docx");
List<Metadata> list = getMetadata(metadata,
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1));
//default behavior (user doesn't specify whether or not to catch embedded exceptions
//is to catch the exception
assertEquals(13, list.size());
Metadata mockNPEMetadata = list.get(10);
assertContains("java.lang.NullPointerException", mockNPEMetadata.get(RecursiveParserWrapper.EMBEDDED_EXCEPTION));
metadata = new Metadata();
metadata.set(Metadata.RESOURCE_NAME_KEY, "test_recursive_embedded_npe.docx");
list = getMetadata(metadata,
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1),
false, null);
//Composite parser swallows caught TikaExceptions, IOExceptions and SAXExceptions
//and just doesn't bother to report that there was an exception.
assertEquals(12, list.size());
}
@Test
public void testPrimaryExcWEmbedded() throws Exception {
//if embedded content is handled and then
//the parser hits an exception in the container document,
//that the first element of the returned list is the container document
//and the second is the embedded content
Metadata metadata = new Metadata();
metadata.set(Metadata.RESOURCE_NAME_KEY, "embedded_then_npe.xml");
ParseContext context = new ParseContext();
Parser wrapped = new AutoDetectParser();
RecursiveParserWrapper wrapper = new RecursiveParserWrapper(wrapped,
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1), true);
String path = "/test-documents/mock/embedded_then_npe.xml";
InputStream stream = null;
boolean npe = false;
try {
stream = RecursiveParserWrapperTest.class.getResourceAsStream(
path);
wrapper.parse(stream, new DefaultHandler(), metadata, context);
} catch (TikaException e) {
if (e.getCause().getClass().equals(NullPointerException.class)) {
npe = true;
}
} finally {
IOUtils.closeQuietly(stream);
}
assertTrue("npe", npe);
List<Metadata> metadataList = wrapper.getMetadata();
assertEquals(2, metadataList.size());
Metadata outerMetadata = metadataList.get(0);
Metadata embeddedMetadata = metadataList.get(1);
assertContains("main_content", outerMetadata.get(RecursiveParserWrapper.TIKA_CONTENT));
assertEquals("embedded_then_npe.xml", outerMetadata.get(TikaMetadataKeys.RESOURCE_NAME_KEY));
assertEquals("Nikolai Lobachevsky", outerMetadata.get("author"));
assertContains("some_embedded_content", embeddedMetadata.get(RecursiveParserWrapper.TIKA_CONTENT));
assertEquals("embed1.xml", embeddedMetadata.get(TikaMetadataKeys.RESOURCE_NAME_KEY));
assertEquals("embeddedAuthor", embeddedMetadata.get("author"));
}
@Test
public void testDigesters() throws Exception {
Metadata metadata = new Metadata();
metadata.set(Metadata.RESOURCE_NAME_KEY, "test_recursive_embedded.docx");
List<Metadata> list = getMetadata(metadata,
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1),
true, new CommonsDigester(100000, CommonsDigester.DigestAlgorithm.MD5));
int i = 0;
Metadata m0 = list.get(0);
Metadata m6 = list.get(6);
String md5Key = "X-TIKA:digest:MD5";
assertEquals("59f626e09a8c16ab6dbc2800c685f772", list.get(0).get(md5Key));
assertEquals("ccdf3882e7e4c2454e28884db9b0a54d", list.get(6).get(md5Key));
assertEquals("a869bf6432ebd14e19fc79416274e0c9", list.get(7).get(md5Key));
}
private List<Metadata> getMetadata(Metadata metadata, ContentHandlerFactory contentHandlerFactory,
boolean catchEmbeddedExceptions,
DigestingParser.Digester digester) throws Exception {
ParseContext context = new ParseContext();
Parser wrapped = new AutoDetectParser();
if (digester != null) {
wrapped = new DigestingParser(wrapped, digester);
}
RecursiveParserWrapper wrapper = new RecursiveParserWrapper(wrapped,
contentHandlerFactory, catchEmbeddedExceptions);
String path = metadata.get(Metadata.RESOURCE_NAME_KEY);
if (path == null) {
path = "/test-documents/test_recursive_embedded.docx";
} else {
path = "/test-documents/" + path;
}
InputStream stream = null;
try {
stream = TikaInputStream.get(RecursiveParserWrapperTest.class.getResource(path).toURI());
wrapper.parse(stream, new DefaultHandler(), metadata, context);
} finally {
IOUtils.closeQuietly(stream);
}
return wrapper.getMetadata();
}
private List<Metadata> getMetadata(Metadata metadata, ContentHandlerFactory contentHandlerFactory)
throws Exception {
return getMetadata(metadata, contentHandlerFactory, true, null);
}
}