/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.parser.microsoft.ooxml;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.sax.TransformerHandler;
import javax.xml.transform.stream.StreamResult;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.InputStream;
import java.io.PrintStream;
import java.io.StringWriter;
import java.text.DecimalFormatSymbols;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.poi.util.LocaleUtil;
import org.apache.tika.TikaTest;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.detect.DefaultDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.exception.EncryptedDocumentException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Office;
import org.apache.tika.metadata.OfficeOpenXMLCore;
import org.apache.tika.metadata.OfficeOpenXMLExtended;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.metadata.TikaMetadataKeys;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.EmptyParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.PasswordProvider;
import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.parser.microsoft.ExcelParserTest;
import org.apache.tika.parser.microsoft.OfficeParser;
import org.apache.tika.parser.microsoft.OfficeParserConfig;
import org.apache.tika.parser.microsoft.WordParserTest;
import org.apache.tika.sax.BodyContentHandler;
import org.junit.Ignore;
import org.junit.Test;
import org.xml.sax.ContentHandler;
public class OOXMLParserTest extends TikaTest {
private Parser parser = new AutoDetectParser();
private InputStream getTestDocument(String name) {
return TikaInputStream.get(OOXMLParserTest.class.getResourceAsStream(
"/test-documents/" + name));
}
@Test
public void testExcel() throws Exception {
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
context.set(Locale.class, Locale.US);
try (InputStream input = getTestDocument("testEXCEL.xlsx")) {
parser.parse(input, handler, metadata, context);
assertEquals(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
metadata.get(Metadata.CONTENT_TYPE));
assertEquals("Simple Excel document", metadata.get(TikaCoreProperties.TITLE));
assertEquals("Keith Bennett", metadata.get(TikaCoreProperties.CREATOR));
assertEquals("Keith Bennett", metadata.get(Metadata.AUTHOR));
String content = handler.toString();
assertContains("Sample Excel Worksheet", content);
assertContains("Numbers and their Squares", content);
assertContains("9", content);
assertNotContained("9.0", content);
assertContains("196", content);
assertNotContained("196.0", content);
assertEquals("false", metadata.get(TikaMetadataKeys.PROTECTED));
}
}
@Test
public void testExcelFormats() throws Exception {
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
context.set(Locale.class, Locale.US);
try (InputStream input = getTestDocument("testEXCEL-formats.xlsx")) {
parser.parse(input, handler, metadata, context);
assertEquals(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
metadata.get(Metadata.CONTENT_TYPE));
String content = handler.toString();
// Number #,##0.00
assertContains("1,599.99", content);
assertContains("-1,599.99", content);
// Currency $#,##0.00;[Red]($#,##0.00)
assertContains("$1,599.99", content);
assertContains("$1,599.99)", content);
// Scientific 0.00E+00
// poi <=3.8beta1 returns 1.98E08, newer versions return 1.98+E08
assertTrue(content.contains("1.98E08") || content.contains("1.98E+08"));
assertTrue(content.contains("-1.98E08") || content.contains("-1.98E+08"));
// Percentage
assertContains("2.50%", content);
// Excel rounds up to 3%, but that requires Java 1.6 or later
if (System.getProperty("java.version").startsWith("1.5")) {
assertContains("2%", content);
} else {
assertContains("3%", content);
}
// Time Format: h:mm
assertContains("6:15", content);
assertContains("18:15", content);
// Date Format: d-mmm-yy
assertContains("17-May-07", content);
// Currency $#,##0.00;[Red]($#,##0.00)
assertContains("$1,599.99", content);
assertContains("($1,599.99)", content);
// Fraction (2.5): # ?/?
assertContains("2 1/2", content);
// Below assertions represent outstanding formatting issues to be addressed
// they are included to allow the issues to be progressed with the Apache POI
// team - See TIKA-103.
/*************************************************************************
// Date Format: m/d/yy
assertContains("03/10/2009", content);
// Date/Time Format
assertContains("19/01/2008 04:35", content);
// Custom Number (0 "dollars and" .00 "cents")
assertContains("19 dollars and .99 cents", content);
// Custom Number ("At" h:mm AM/PM "on" dddd mmmm d"," yyyy)
assertContains("At 4:20 AM on Thursday May 17, 2007", content);
**************************************************************************/
}
}
@Test
@Ignore("OOXML-Strict not currently supported by POI, see #57699")
public void testExcelStrict() throws Exception {
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
context.set(Locale.class, Locale.US);
try (InputStream input = getTestDocument("testEXCEL.strict.xlsx")) {
parser.parse(input, handler, metadata, context);
assertEquals(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
metadata.get(Metadata.CONTENT_TYPE));
assertEquals("Sample Spreadsheet", metadata.get(TikaCoreProperties.TITLE));
assertEquals("Nick Burch", metadata.get(TikaCoreProperties.CREATOR));
assertEquals("Spreadsheet for testing", metadata.get(TikaCoreProperties.DESCRIPTION));
String content = handler.toString();
assertContains("Test spreadsheet", content);
assertContains("This one is red", content);
assertContains("cb=10", content);
assertNotContained("10.0", content);
assertContains("cb=sum", content);
assertNotContained("13.0", content);
assertEquals("false", metadata.get(TikaMetadataKeys.PROTECTED));
}
}
/**
* We have a number of different powerpoint files,
* such as presentation, macro-enabled etc
*/
@Test
public void testPowerPoint() throws Exception {
String[] extensions = new String[]{
"pptx", "pptm", "ppsm", "ppsx", "potm"
//"thmx", // TIKA-418: Will be supported in POI 3.7 beta 2
//"xps" // TIKA-418: Not yet supported by POI
};
String[] mimeTypes = new String[]{
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.ms-powerpoint.presentation.macroenabled.12",
"application/vnd.ms-powerpoint.slideshow.macroenabled.12",
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
"application/vnd.ms-powerpoint.template.macroenabled.12"
};
for (int i = 0; i < extensions.length; i++) {
String extension = extensions[i];
String filename = "testPPT." + extension;
Parser parser = new AutoDetectParser();
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
try (InputStream input = getTestDocument(filename)) {
parser.parse(input, handler, metadata, context);
assertEquals(
"Mime-type checking for " + filename,
mimeTypes[i],
metadata.get(Metadata.CONTENT_TYPE));
assertEquals("Attachment Test", metadata.get(TikaCoreProperties.TITLE));
assertEquals("Rajiv", metadata.get(TikaCoreProperties.CREATOR));
assertEquals("Rajiv", metadata.get(Metadata.AUTHOR));
String content = handler.toString();
// Theme files don't have the text in them
if (extension.equals("thmx")) {
assertEquals("", content);
} else {
assertTrue(
"Text missing for " + filename + "\n" + content,
content.contains("Attachment Test")
);
assertTrue(
"Text missing for " + filename + "\n" + content,
content.contains("This is a test file data with the same content")
);
assertTrue(
"Text missing for " + filename + "\n" + content,
content.contains("content parsing")
);
assertTrue(
"Text missing for " + filename + "\n" + content,
content.contains("Different words to test against")
);
assertTrue(
"Text missing for " + filename + "\n" + content,
content.contains("Mystery")
);
}
}
}
}
/**
* Test that the metadata is already extracted when the body is processed.
* See TIKA-1109
*/
@Test
public void testPowerPointMetadataEarly() throws Exception {
String[] extensions = new String[]{
"pptx", "pptm", "ppsm", "ppsx", "potm"
//"thmx", // TIKA-418: Will be supported in POI 3.7 beta 2
//"xps" // TIKA-418: Not yet supported by POI
};
final String[] mimeTypes = new String[]{
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.ms-powerpoint.presentation.macroenabled.12",
"application/vnd.ms-powerpoint.slideshow.macroenabled.12",
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
"application/vnd.ms-powerpoint.template.macroenabled.12"
};
for (int i = 0; i < extensions.length; i++) {
String extension = extensions[i];
final String filename = "testPPT." + extension;
Parser parser = new AutoDetectParser();
final Metadata metadata = new Metadata();
// Allow the value to be access from the inner class
final int currentI = i;
ContentHandler handler = new BodyContentHandler() {
public void startDocument() {
assertEquals(
"Mime-type checking for " + filename,
mimeTypes[currentI],
metadata.get(Metadata.CONTENT_TYPE));
assertEquals("Attachment Test", metadata.get(TikaCoreProperties.TITLE));
assertEquals("Rajiv", metadata.get(TikaCoreProperties.CREATOR));
assertEquals("Rajiv", metadata.get(Metadata.AUTHOR));
}
};
ParseContext context = new ParseContext();
try (InputStream input = getTestDocument(filename)) {
parser.parse(input, handler, metadata, context);
}
}
}
/**
* For the PowerPoint formats we don't currently support, ensure that
* we don't break either
*/
@Test
public void testUnsupportedPowerPoint() throws Exception {
String[] extensions = new String[]{"xps", "thmx"};
String[] mimeTypes = new String[]{
"application/vnd.ms-xpsdocument",
"application/vnd.openxmlformats-officedocument" // Is this right?
};
for (int i = 0; i < extensions.length; i++) {
String extension = extensions[i];
String filename = "testPPT." + extension;
Parser parser = new AutoDetectParser();
Metadata metadata = new Metadata();
metadata.set(Metadata.RESOURCE_NAME_KEY, filename);
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
try (InputStream input = getTestDocument(filename)) {
parser.parse(input, handler, metadata, context);
// Should get the metadata
assertEquals(
"Mime-type checking for " + filename,
mimeTypes[i],
metadata.get(Metadata.CONTENT_TYPE));
// But that's about it
}
}
}
/**
* Test the plain text output of the Word converter
*
* @throws Exception
*/
@Test
public void testWord() throws Exception {
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
try (InputStream input = getTestDocument("testWORD.docx")) {
parser.parse(input, handler, metadata, context);
assertEquals(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
metadata.get(Metadata.CONTENT_TYPE));
assertEquals("Sample Word Document", metadata.get(TikaCoreProperties.TITLE));
assertEquals("Keith Bennett", metadata.get(TikaCoreProperties.CREATOR));
assertEquals("Keith Bennett", metadata.get(Metadata.AUTHOR));
assertTrue(handler.toString().contains("Sample Word Document"));
}
}
/**
* Test the plain text output of the Word converter
*
* @throws Exception
*/
@Test
public void testWordFootnote() throws Exception {
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
try (InputStream input = getTestDocument("footnotes.docx")) {
parser.parse(input, handler, metadata, context);
assertEquals(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
metadata.get(Metadata.CONTENT_TYPE));
assertTrue(handler.toString().contains("snoska"));
}
}
/**
* Test that the word converter is able to generate the
* correct HTML for the document
*/
@Test
public void testWordHTML() throws Exception {
XMLResult result = getXML("testWORD.docx");
String xml = result.xml;
Metadata metadata = result.metadata;
assertEquals(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
metadata.get(Metadata.CONTENT_TYPE));
assertEquals("Sample Word Document", metadata.get(TikaCoreProperties.TITLE));
assertEquals("Keith Bennett", metadata.get(TikaCoreProperties.CREATOR));
assertEquals("Keith Bennett", metadata.get(Metadata.AUTHOR));
assertTrue(xml.contains("Sample Word Document"));
// Check that custom headings came through
assertTrue(xml.contains("<h1 class=\"title\">"));
// Regular headings
assertTrue(xml.contains("<h1>Heading Level 1</h1>"));
assertTrue(xml.contains("<h2>Heading Level 2</h2>"));
// Headings with anchor tags in them
assertTrue(xml.contains("<h3><a name=\"OnLevel3\" />Heading Level 3</h3>"));
// Bold and italic
assertTrue(xml.contains("<b>BOLD</b>"));
assertTrue(xml.contains("<i>ITALIC</i>"));
// Table
assertTrue(xml.contains("<table>"));
assertTrue(xml.contains("<td>"));
// Links
assertTrue(xml.contains("<a href=\"http://tika.apache.org/\">Tika</a>"));
// Anchor links
assertContains("<a href=\"#OnMainHeading\">The Main Heading Bookmark</a>", xml);
// Paragraphs with other styles
assertTrue(xml.contains("<p class=\"signature\">This one"));
result = getXML("testWORD_3imgs.docx");
xml = result.xml;
// Images 2-4 (there is no 1!)
assertTrue("Image not found in:\n" + xml, xml.contains("<img src=\"embedded:image2.png\" alt=\"A description...\" />"));
assertTrue("Image not found in:\n" + xml, xml.contains("<img src=\"embedded:image3.jpeg\" alt=\"A description...\" />"));
assertTrue("Image not found in:\n" + xml, xml.contains("<img src=\"embedded:image4.png\" alt=\"A description...\" />"));
// Text too
assertTrue(xml.contains("<p>The end!</p>"));
// TIKA-692: test document containing multiple
// character runs within a bold tag:
xml = getXML("testWORD_bold_character_runs.docx").xml;
// Make sure bold text arrived as single
// contiguous string even though Word parser
// handled this as 3 character runs
assertTrue("Bold text wasn't contiguous: " + xml, xml.contains("F<b>oob</b>a<b>r</b>"));
// TIKA-692: test document containing multiple
// character runs within a bold tag:
xml = getXML("testWORD_bold_character_runs2.docx").xml;
// Make sure bold text arrived as single
// contiguous string even though Word parser
// handled this as 3 character runs
assertTrue("Bold text wasn't contiguous: " + xml, xml.contains("F<b>oob</b>a<b>r</b>"));
}
/**
* Test that we can extract image from docx header
*/
@Test
public void testWordPicturesInHeader() throws Exception {
List<Metadata> metadataList = getRecursiveMetadata("headerPic.docx");
assertEquals(2, metadataList.size());
Metadata m = metadataList.get(0);
String mainContent = m.get(RecursiveParserWrapper.TIKA_CONTENT);
assertEquals(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
m.get(Metadata.CONTENT_TYPE));
// Check that custom headings came through
assertTrue(mainContent.contains("<img"));
}
@Test
@Ignore("need to add links in xhtml")
public void testPicturesInVariousPlaces() throws Exception {
//test that images are actually extracted from
//headers, footers, comments, endnotes, footnotes
List<Metadata> metadataList = getRecursiveMetadata("testWORD_embedded_pics.docx");
//only process embedded resources once
assertEquals(3, metadataList.size());
String content = metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT);
for (int i = 1; i < 4; i++) {
assertContains("header"+i+"_pic", content);
assertContains("footer"+i+"_pic", content);
}
assertContains("body_pic.jpg", content);
assertContains("sdt_pic.jpg", content);
assertContains("deeply_embedded_pic", content);
assertContains("deleted_pic", content);//TODO: don't extract this
assertContains("footnotes_pic", content);
assertContains("comments_pic", content);
assertContains("endnotes_pic", content);
// assertContains("sdt2_pic.jpg", content);//name of file is not stored in image-sdt
assertContainsCount("<img src=", content, 14);
}
/**
* Documents with some sheets are protected, but not all.
* See TIKA-364.
*/
@Test
public void testProtectedExcelSheets() throws Exception {
Parser parser = new AutoDetectParser();
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
try (InputStream input = OOXMLParserTest.class
.getResourceAsStream("/test-documents/protectedSheets.xlsx")) {
parser.parse(input, handler, metadata, context);
assertEquals(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
metadata.get(Metadata.CONTENT_TYPE));
assertEquals("true", metadata.get(TikaMetadataKeys.PROTECTED));
}
}
/**
* An excel document which is password protected.
* See TIKA-437.
*/
@Test
public void testProtectedExcelFile() throws Exception {
Parser parser = new AutoDetectParser();
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
try (InputStream input = getTestDocument("protectedFile.xlsx")) {
parser.parse(input, handler, metadata, context);
assertEquals(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
metadata.get(Metadata.CONTENT_TYPE));
assertEquals("true", metadata.get(TikaMetadataKeys.PROTECTED));
String content = handler.toString();
assertContains("Office", content);
}
}
/**
* Test docx without headers
* TIKA-633
*/
@Test
public void testNullHeaders() throws Exception {
Parser parser = new AutoDetectParser();
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
try (InputStream input = getTestDocument("NullHeader.docx")) {
parser.parse(input, handler, metadata, context);
assertEquals("Should have found some text", false, handler.toString().isEmpty());
}
}
@Test
public void testVarious() throws Exception {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
try (InputStream stream = OOXMLParserTest.class.getResourceAsStream(
"/test-documents/testWORD_various.docx")) {
new AutoDetectParser().parse(stream, handler, metadata, new ParseContext());
}
String content = handler.toString();
//content = content.replaceAll("\\s+"," ");
assertContains("Footnote appears here", content);
assertContains("This is a footnote.", content);
assertContains("This is the header text.", content);
assertContains("This is the footer text.", content);
assertContains("Here is a text box", content);
assertContains("Bold", content);
assertContains("italic", content);
assertContains("underline", content);
assertContains("superscript", content);
assertContains("subscript", content);
assertContains("Here is a citation:", content);
assertContains("Figure 1 This is a caption for Figure 1", content);
assertContains("(Kramer)", content);
assertContains("Row 1 Col 1 Row 1 Col 2 Row 1 Col 3 Row 2 Col 1 Row 2 Col 2 Row 2 Col 3", content.replaceAll("\\s+", " "));
assertContains("Row 1 column 1 Row 2 column 1 Row 1 column 2 Row 2 column 2", content.replaceAll("\\s+", " "));
assertContains("This is a hyperlink", content);
assertContains("Here is a list:", content);
for (int row = 1; row <= 3; row++) {
//assertContains("·\tBullet " + row, content);
//assertContains("\u00b7\tBullet " + row, content);
assertContains("Bullet " + row, content);
}
assertContains("Here is a numbered list:", content);
for (int row = 1; row <= 3; row++) {
//assertContains(row + ")\tNumber bullet " + row, content);
//assertContains(row + ") Number bullet " + row, content);
// TODO: OOXMLExtractor fails to number the bullets:
assertContains("Number bullet " + row, content);
}
for (int row = 1; row <= 2; row++) {
for (int col = 1; col <= 3; col++) {
assertContains("Row " + row + " Col " + col, content);
}
}
assertContains("Keyword1 Keyword2", content);
assertEquals("Keyword1 Keyword2",
metadata.get(Metadata.KEYWORDS));
assertContains("Subject is here", content);
// TODO: Remove subject in Tika 2.0
assertEquals("Subject is here",
metadata.get(Metadata.SUBJECT));
assertEquals("Subject is here",
metadata.get(OfficeOpenXMLCore.SUBJECT));
assertContains("Suddenly some Japanese text:", content);
// Special version of (GHQ)
assertContains("\uff08\uff27\uff28\uff31\uff09", content);
// 6 other characters
assertContains("\u30be\u30eb\u30b2\u3068\u5c3e\u5d0e\u3001\u6de1\u3005\u3068\u6700\u671f", content);
assertContains("And then some Gothic text:", content);
assertContains("\uD800\uDF32\uD800\uDF3f\uD800\uDF44\uD800\uDF39\uD800\uDF43\uD800\uDF3A", content);
}
@Test
public void testVariousPPTX() throws Exception {
Metadata metadata = new Metadata();
String xml = getXML("testPPT_various.pptx", metadata).xml;
assertContains("<p>Footnote appears here", xml);
assertContains("<p>[1] This is a footnote.", xml);
assertContains("<p>This is the header text.</p>", xml);
assertContains("<p>This is the footer text.</p>", xml);
assertContains("<p>Here is a text box</p>", xml);
assertContains("<p>Bold", xml);
assertContains("italic underline superscript subscript", xml);
assertContains("<p>Here is a citation:", xml);
assertContains("Figure 1 This is a caption for Figure 1", xml);
assertContains("(Kramer)", xml);
assertContains("<table><tr>\t<td>Row 1 Col 1</td>", xml);
assertContains("<td>Row 2 Col 2</td>\t<td>Row 2 Col 3</td></tr>", xml);
assertContains("<p>Row 1 column 1</p>", xml);
assertContains("<p>Row 2 column 2</p>", xml);
assertContains("<p><a href=\"http://tika.apache.org/\">This is a hyperlink</a>", xml);
assertContains("<p>Here is a list:", xml);
for (int row = 1; row <= 3; row++) {
//assertContains("·\tBullet " + row, content);
//assertContains("\u00b7\tBullet " + row, content);
assertContains("<p>Bullet " + row, xml);
}
assertContains("Here is a numbered list:", xml);
for (int row = 1; row <= 3; row++) {
//assertContains(row + ")\tNumber bullet " + row, content);
//assertContains(row + ") Number bullet " + row, content);
// TODO: OOXMLExtractor fails to number the bullets:
assertContains("<p>Number bullet " + row, xml);
}
for (int row = 1; row <= 2; row++) {
for (int col = 1; col <= 3; col++) {
assertContains("Row " + row + " Col " + col, xml);
}
}
assertContains("Keyword1 Keyword2", xml);
assertEquals("Keyword1 Keyword2",
metadata.get(Metadata.KEYWORDS));
assertContains("Subject is here", xml);
// TODO: Remove subject in Tika 2.0
assertEquals("Subject is here",
metadata.get(Metadata.SUBJECT));
assertEquals("Subject is here",
metadata.get(OfficeOpenXMLCore.SUBJECT));
assertContains("Suddenly some Japanese text:", xml);
// Special version of (GHQ)
assertContains("\uff08\uff27\uff28\uff31\uff09", xml);
// 6 other characters
assertContains("\u30be\u30eb\u30b2\u3068\u5c3e\u5d0e\u3001\u6de1\u3005\u3068\u6700\u671f", xml);
assertContains("And then some Gothic text:", xml);
assertContains("\uD800\uDF32\uD800\uDF3f\uD800\uDF44\uD800\uDF39\uD800\uDF43\uD800\uDF3A", xml);
}
@Test
public void testCommentPPTX() throws Exception {
XMLResult r = getXML("testPPT_comment.pptx");
assertContains("<p class=\"slide-comment\"><b>Allison, Timothy B. (ATB)", r.xml);
}
@Test
public void testMasterFooter() throws Exception {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
try (InputStream stream = OOXMLParserTest.class.getResourceAsStream(
"/test-documents/testPPT_masterFooter.pptx")) {
new AutoDetectParser().parse(stream, handler, metadata, new ParseContext());
}
String content = handler.toString();
assertContains("Master footer is here", content);
}
/**
* TIKA-712 Master Slide Text from PPT and PPTX files
* should be extracted too
*/
@Test
public void testMasterText() throws Exception {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
try (InputStream stream = OOXMLParserTest.class.getResourceAsStream(
"/test-documents/testPPT_masterText.pptx")) {
new AutoDetectParser().parse(stream, handler, metadata, new ParseContext());
}
String content = handler.toString();
assertContains("Text that I added to the master slide", content);
}
@Test
public void testMasterText2() throws Exception {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
try (InputStream stream = OOXMLParserTest.class.getResourceAsStream(
"/test-documents/testPPT_masterText2.pptx")) {
new AutoDetectParser().parse(stream, handler, metadata, new ParseContext());
}
String content = handler.toString();
assertContains("Text that I added to the master slide", content);
}
@Test
public void testWordArt() throws Exception {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
try (InputStream stream = OOXMLParserTest.class.getResourceAsStream(
"/test-documents/testWordArt.pptx")) {
new AutoDetectParser().parse(stream, handler, metadata, new ParseContext());
}
String content = handler.toString();
assertContains("Here is some red word Art", content);
}
/**
* Ensures that custom OOXML properties are extracted
*/
@Test
public void testExcelCustomProperties() throws Exception {
Metadata metadata = new Metadata();
try (InputStream input = OOXMLParserTest.class.getResourceAsStream(
"/test-documents/testEXCEL_custom_props.xlsx")) {
ContentHandler handler = new BodyContentHandler(-1);
ParseContext context = new ParseContext();
context.set(Locale.class, Locale.US);
new OOXMLParser().parse(input, handler, metadata, context);
}
assertEquals(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
metadata.get(Metadata.CONTENT_TYPE));
assertEquals(null, metadata.get(TikaCoreProperties.CREATOR));
assertEquals(null, metadata.get(TikaCoreProperties.MODIFIER));
assertEquals("2006-09-12T15:06:44Z", metadata.get(TikaCoreProperties.CREATED));
assertEquals("2006-09-12T15:06:44Z", metadata.get(Metadata.CREATION_DATE));
assertEquals("2011-08-22T14:24:38Z", metadata.get(Metadata.LAST_MODIFIED));
assertEquals("2011-08-22T14:24:38Z", metadata.get(TikaCoreProperties.MODIFIED));
assertEquals("2011-08-22T14:24:38Z", metadata.get(Metadata.DATE));
assertEquals("Microsoft Excel", metadata.get(Metadata.APPLICATION_NAME));
assertEquals("Microsoft Excel", metadata.get(OfficeOpenXMLExtended.APPLICATION));
assertEquals("true", metadata.get("custom:myCustomBoolean"));
assertEquals("3", metadata.get("custom:myCustomNumber"));
assertEquals("MyStringValue", metadata.get("custom:MyCustomString"));
assertEquals("2010-12-30T22:00:00Z", metadata.get("custom:MyCustomDate"));
assertEquals("2010-12-29T22:00:00Z", metadata.get("custom:myCustomSecondDate"));
}
@Test
public void testWordCustomProperties() throws Exception {
Metadata metadata = new Metadata();
try (InputStream input = OOXMLParserTest.class.getResourceAsStream(
"/test-documents/testWORD_custom_props.docx")) {
ContentHandler handler = new BodyContentHandler(-1);
ParseContext context = new ParseContext();
context.set(Locale.class, Locale.US);
new OOXMLParser().parse(input, handler, metadata, context);
}
assertEquals(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
metadata.get(Metadata.CONTENT_TYPE));
assertEquals("EJ04325S", metadata.get(TikaCoreProperties.CREATOR));
assertEquals("Etienne Jouvin", metadata.get(TikaCoreProperties.MODIFIER));
assertEquals("Etienne Jouvin", metadata.get(Metadata.LAST_AUTHOR));
assertEquals("2011-07-29T16:52:00Z", metadata.get(TikaCoreProperties.CREATED));
assertEquals("2011-07-29T16:52:00Z", metadata.get(Metadata.CREATION_DATE));
assertEquals("2012-01-03T22:14:00Z", metadata.get(TikaCoreProperties.MODIFIED));
assertEquals("2012-01-03T22:14:00Z", metadata.get(Metadata.DATE));
assertEquals("Microsoft Office Word", metadata.get(Metadata.APPLICATION_NAME));
assertEquals("Microsoft Office Word", metadata.get(OfficeOpenXMLExtended.APPLICATION));
assertEquals("1", metadata.get(Office.PAGE_COUNT));
assertEquals("2", metadata.get(Office.WORD_COUNT));
assertEquals("My Title", metadata.get(TikaCoreProperties.TITLE));
assertEquals("My Keyword", metadata.get(TikaCoreProperties.KEYWORDS));
assertEquals("Normal.dotm", metadata.get(Metadata.TEMPLATE));
assertEquals("Normal.dotm", metadata.get(OfficeOpenXMLExtended.TEMPLATE));
// TODO: Remove subject in Tika 2.0
assertEquals("My subject", metadata.get(Metadata.SUBJECT));
assertEquals("My subject", metadata.get(OfficeOpenXMLCore.SUBJECT));
assertEquals("EDF-DIT", metadata.get(TikaCoreProperties.PUBLISHER));
assertEquals("true", metadata.get("custom:myCustomBoolean"));
assertEquals("3", metadata.get("custom:myCustomNumber"));
assertEquals("MyStringValue", metadata.get("custom:MyCustomString"));
assertEquals("2010-12-30T23:00:00Z", metadata.get("custom:MyCustomDate"));
assertEquals("2010-12-29T22:00:00Z", metadata.get("custom:myCustomSecondDate"));
}
@Test
public void testPowerPointCustomProperties() throws Exception {
Metadata metadata = new Metadata();
try (InputStream input = OOXMLParserTest.class.getResourceAsStream(
"/test-documents/testPPT_custom_props.pptx")) {
ContentHandler handler = new BodyContentHandler(-1);
ParseContext context = new ParseContext();
context.set(Locale.class, Locale.US);
new OOXMLParser().parse(input, handler, metadata, context);
}
assertEquals(
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
metadata.get(Metadata.CONTENT_TYPE));
assertEquals("JOUVIN ETIENNE", metadata.get(TikaCoreProperties.CREATOR));
assertEquals("EJ04325S", metadata.get(TikaCoreProperties.MODIFIER));
assertEquals("EJ04325S", metadata.get(Metadata.LAST_AUTHOR));
assertEquals("2011-08-22T13:30:53Z", metadata.get(TikaCoreProperties.CREATED));
assertEquals("2011-08-22T13:30:53Z", metadata.get(Metadata.CREATION_DATE));
assertEquals("2011-08-22T13:32:49Z", metadata.get(TikaCoreProperties.MODIFIED));
assertEquals("2011-08-22T13:32:49Z", metadata.get(Metadata.DATE));
assertEquals("1", metadata.get(Office.SLIDE_COUNT));
assertEquals("3", metadata.get(Office.WORD_COUNT));
assertEquals("Test extraction properties pptx", metadata.get(TikaCoreProperties.TITLE));
assertEquals("true", metadata.get("custom:myCustomBoolean"));
assertEquals("3", metadata.get("custom:myCustomNumber"));
assertEquals("MyStringValue", metadata.get("custom:MyCustomString"));
assertEquals("2010-12-30T22:00:00Z", metadata.get("custom:MyCustomDate"));
assertEquals("2010-12-29T22:00:00Z", metadata.get("custom:myCustomSecondDate"));
}
// TIKA-989:
@Test
public void testEmbeddedPDF() throws Exception {
Metadata metadata = new Metadata();
StringWriter sw = new StringWriter();
SAXTransformerFactory factory = (SAXTransformerFactory)
SAXTransformerFactory.newInstance();
TransformerHandler handler = factory.newTransformerHandler();
handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
handler.getTransformer().setOutputProperty(OutputKeys.INDENT, "no");
handler.setResult(new StreamResult(sw));
try (InputStream input = OOXMLParserTest.class.getResourceAsStream(
"/test-documents/testWORD_embedded_pdf.docx")) {
new OOXMLParser().parse(input, handler, metadata, new ParseContext());
}
String xml = sw.toString();
int i = xml.indexOf("Here is the pdf file:");
int j = xml.indexOf("<div class=\"embedded\" id=\"rId5\"/>");
int k = xml.indexOf("Bye Bye");
int l = xml.indexOf("<div class=\"embedded\" id=\"rId6\"/>");
int m = xml.indexOf("Bye for real.");
assertTrue(i != -1);
assertTrue(j != -1);
assertTrue(k != -1);
assertTrue(l != -1);
assertTrue(m != -1);
assertTrue(i < j);
assertTrue(j < k);
assertTrue(k < l);
assertTrue(l < m);
}
// TIKA-997:
@Test
public void testEmbeddedZipInPPTX() throws Exception {
String xml = getXML("test_embedded_zip.pptx").xml;
int h = xml.indexOf("<div class=\"embedded\" id=\"slide1_rId3\" />");
int i = xml.indexOf("Send me a note");
int j = xml.indexOf("<div class=\"embedded\" id=\"slide2_rId4\" />");
int k = xml.indexOf("<p>No title</p>");
assertTrue(h != -1);
assertTrue(i != -1);
assertTrue(j != -1);
assertTrue(k != -1);
assertTrue(h < i);
assertTrue(i < j);
assertTrue(j < k);
}
// TIKA-1006
@Test
public void testWordNullStyle() throws Exception {
String xml = getXML("testWORD_null_style.docx").xml;
assertContains("Test av styrt dokument", xml);
}
/**
* TIKA-1044 - Handle word documents where parts of the
* text have no formatting or styles applied to them
*/
@Test
public void testNoFormat() throws Exception {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
try (InputStream stream = WordParserTest.class.getResourceAsStream(
"/test-documents/testWORD_no_format.docx")) {
new OOXMLParser().parse(stream, handler, metadata, new ParseContext());
}
String content = handler.toString();
assertContains("This is a piece of text that causes an exception", content);
}
// TIKA-1005:
@Test
public void testTextInsideTextBox() throws Exception {
String xml = getXML("testWORD_text_box.docx").xml;
assertContains("This text is directly in the body of the document.", xml);
assertContains("This text is inside of a text box in the body of the document.", xml);
assertContains("This text is inside of a text box in the header of the document.", xml);
assertContains("This text is inside of a text box in the footer of the document.", xml);
}
//TIKA-2346
@Test
public void testTurningOffTextBoxExtraction() throws Exception {
ParseContext pc = new ParseContext();
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
officeParserConfig.setIncludeShapeBasedContent(false);
pc.set(OfficeParserConfig.class, officeParserConfig);
String xml = getXML("testWORD_text_box.docx", pc).xml;
assertContains("This text is directly in the body of the document.", xml);
assertNotContained("This text is inside of a text box in the body of the document.", xml);
assertNotContained("This text is inside of a text box in the header of the document.", xml);
assertNotContained("This text is inside of a text box in the footer of the document.", xml);
}
// TIKA-1032:
@Test
public void testEmbeddedPPTXTwoSlides() throws Exception {
String xml = getXML("testPPT_embedded_two_slides.pptx").xml;
assertContains("<div class=\"embedded\" id=\"slide1_rId7\" />", xml);
assertContains("<div class=\"embedded\" id=\"slide2_rId7\" />", xml);
}
/**
* Test for missing text described in
* <a href="https://issues.apache.org/jira/browse/TIKA-1130">TIKA-1130</a>.
* and TIKA-1317
*/
@Test
public void testMissingText() throws Exception {
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
try (InputStream input = getTestDocument("testWORD_missing_text.docx")) {
parser.parse(input, handler, metadata, context);
assertEquals(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
metadata.get(Metadata.CONTENT_TYPE));
assertContains("BigCompany", handler.toString());
assertContains("Seasoned", handler.toString());
assertContains("Rich_text_in_cell", handler.toString());
}
}
//TIKA-1100:
@Test
public void testExcelTextBox() throws Exception {
XMLResult r = getXML("testEXCEL_textbox.xlsx", parser);
assertContains("some autoshape", r.xml);
}
//TIKA-2346
@Test
public void testTurningOffTextBoxExtractionExcel() throws Exception {
ParseContext pc = new ParseContext();
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
officeParserConfig.setIncludeShapeBasedContent(false);
pc.set(OfficeParserConfig.class, officeParserConfig);
String xml = getXML("testEXCEL_textbox.xlsx", pc).xml;
assertNotContained("autoshape", xml);
}
//TIKA-792; with room for future missing bean tests
@Test
public void testWordMissingOOXMLBeans() throws Exception {
//If a bean is missing, POI prints stack trace to stderr
String[] fileNames = new String[]{
"testWORD_missing_ooxml_bean1.docx",//TIKA-792
};
PrintStream origErr = System.err;
for (String fileName : fileNames) {
Metadata metadata = new Metadata();
ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
InputStream input = getTestDocument(fileName);
//grab stderr
ByteArrayOutputStream errContent = new ByteArrayOutputStream();
System.setErr(new PrintStream(errContent, true, UTF_8.name()));
parser.parse(input, handler, metadata, context);
//return stderr
System.setErr(origErr);
String err = errContent.toString(UTF_8.name());
assertTrue(err.length() == 0);
input.close();
}
}
//TIKA-817
@Test
public void testPPTXAutodate() throws Exception {
//Following POI-52368, the stored date is extracted,
//not the auto-generated date.
XMLResult result = getXML("testPPT_autodate.pptx");
assertContains("<p>Now</p>\n" +
"<p>2011-12-19 10:20:04 AM</p>\n", result.xml);
}
@Test
public void testDOCXThumbnail() throws Exception {
String xml = getXML("testDOCX_Thumbnail.docx").xml;
int a = xml.indexOf("This file contains a thumbnail");
int b = xml.indexOf("<div class=\"embedded\" id=\"/docProps/thumbnail.emf\" />");
assertTrue(a != -1);
assertTrue(b != -1);
assertTrue(a < b);
}
@Test
public void testXLSXThumbnail() throws Exception {
String xml = getXML("testXLSX_Thumbnail.xlsx").xml;
int a = xml.indexOf("This file contains an embedded thumbnail by default");
int b = xml.indexOf("<div class=\"embedded\" id=\"/docProps/thumbnail.wmf\" />");
assertTrue(a != -1);
assertTrue(b != -1);
assertTrue(a < b);
}
@Test
public void testPPTXThumbnail() throws Exception {
String xml = getXML("testPPTX_Thumbnail.pptx").xml;
int a = xml.indexOf("<body><div class=\"slide-content\"><p>This file contains an embedded thumbnail");
int b = xml.indexOf("<div class=\"embedded\" id=\"/docProps/thumbnail.jpeg\" />");
assertTrue(a != -1);
assertTrue(b != -1);
assertTrue(a < b);
}
@Test
public void testEncrypted() throws Exception {
Map<String, String> tests = new HashMap<String, String>();
tests.put("testWORD_protected_passtika.docx",
"This is an encrypted Word 2007 File");
tests.put("testPPT_protected_passtika.pptx",
"This is an encrypted PowerPoint 2007 slide.");
tests.put("testEXCEL_protected_passtika.xlsx",
"This is an Encrypted Excel spreadsheet.");
Parser parser = new AutoDetectParser();
Metadata m = new Metadata();
PasswordProvider passwordProvider = new PasswordProvider() {
@Override
public String getPassword(Metadata metadata) {
return "tika";
}
};
ParseContext passwordContext = new ParseContext();
passwordContext.set(org.apache.tika.parser.PasswordProvider.class, passwordProvider);
for (Map.Entry<String, String> e : tests.entrySet()) {
try (InputStream is = getTestDocument(e.getKey())) {
ContentHandler handler = new BodyContentHandler();
parser.parse(is, handler, m, passwordContext);
assertContains(e.getValue(), handler.toString());
}
}
ParseContext context = new ParseContext();
//now try with no password
for (Map.Entry<String, String> e : tests.entrySet()) {
boolean exc = false;
try (InputStream is = getTestDocument(e.getKey())) {
ContentHandler handler = new BodyContentHandler();
parser.parse(is, handler, m, context);
} catch (EncryptedDocumentException ex) {
exc = true;
}
assertTrue(exc);
}
}
@Test
public void testDOCXParagraphNumbering() throws Exception {
String xml = getXML("testWORD_numbered_list.docx").xml;
assertContains("1) This", xml);
assertContains("a) Is", xml);
assertContains("i) A multi", xml);
assertContains("ii) Level", xml);
assertContains("1. Within cell 1", xml);
assertContains("b. Cell b", xml);
assertContains("iii) List", xml);
assertContains("2) foo", xml);
assertContains("ii) baz", xml);
assertContains("ii) foo", xml);
assertContains("II. bar", xml);
assertContains("6. six", xml);
assertContains("7. seven", xml);
assertContains("a. seven a", xml);
assertContains("e. seven e", xml);
assertContains("2. A ii 2", xml);
assertContains("3. page break list 3", xml);
assertContains("Some-1-CrazyFormat Greek numbering with crazy format - alpha", xml);
assertContains("1.1.1. 1.1.1", xml);
assertContains("1.1. 1.2->1.1 //set the value", xml);
//TODO: comment is not being extracted!
//assertContains("add a list here", xml);
}
@Test
public void testDOCXOverrideParagraphNumbering() throws Exception {
String xml = getXML("testWORD_override_list_numbering.docx").xml;
//Test 1
assertContains("<p>1.1.1.1...1 1.1.1.1...1</p>", xml);
assertContains("1st.2.3someText 1st.2.3someText", xml);
assertContains("1st.2.2someOtherText.1 1st.2.2someOtherText.1", xml);
assertContains("5th 5th", xml);
//Test 2
assertContains("1.a.I 1.a.I", xml);
//test no reset because level 2 is not sufficient to reset
assertContains("<p>1.b.III 1.b.III</p>", xml);
//test restarted because of level 0's increment to 2
assertContains("2.a.I 2.a.I", xml);
//test handling of skipped level
assertContains("<p>2.b 2.b</p>", xml);
//Test 3
assertContains("(1)) (1))", xml);
//tests start level 1 at 17 and
assertContains("2.17 2.17", xml);
//tests that isLegal turns everything into decimal
assertContains("2.18.2.1 2.18.2.1", xml);
assertContains("<p>2 2</p>", xml);
//Test4
assertContains("<p>1 1</p>", xml);
assertContains("<p>A A</p>", xml);
assertContains("<p>B B</p>", xml);
//this tests overrides
assertContains("<p>C C</p>", xml);
assertContains("<p>4 4</p>", xml);
//Test5
assertContains(">00 00", xml);
assertContains(">01 01", xml);
assertContains(">01. 01.", xml);
assertContains(">01..1 01..1", xml);
assertContains(">02 02", xml);
}
@Test
public void testExcelHeaderAndFooterExtraction() throws Exception {
XMLResult xml = getXML("testEXCEL_headers_footers.xlsx");
assertEquals(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
xml.metadata.get(Metadata.CONTENT_TYPE));
assertEquals("Internal spreadsheet", xml.metadata.get(TikaCoreProperties.TITLE));
assertEquals("Aeham Abushwashi", xml.metadata.get(TikaCoreProperties.CREATOR));
assertEquals("Aeham Abushwashi", xml.metadata.get(Metadata.AUTHOR));
String content = xml.xml;
assertContains("John Smith1", content);
assertContains("John Smith50", content);
assertContains("1 Corporate HQ", content);
assertContains("Header - Corporate Spreadsheet", content);
assertContains("Header - For Internal Use Only", content);
assertContains("Header - Author: John Smith", content);
assertContains("Footer - Corporate Spreadsheet", content);
assertContains("Footer - For Internal Use Only", content);
assertContains("Footer - Author: John Smith", content);
}
@Test
public void testMultiAuthorsManagers() throws Exception {
XMLResult r = getXML("testWORD_multi_authors.docx");
String[] authors = r.metadata.getValues(TikaCoreProperties.CREATOR);
assertEquals(3, authors.length);
assertEquals("author2", authors[1]);
String[] managers = r.metadata.getValues(OfficeOpenXMLExtended.MANAGER);
assertEquals(2, managers.length);
assertEquals("manager1", managers[0]);
assertEquals("manager2", managers[1]);
}
@Test
public void testHyperlinksInXLSX() throws Exception {
String xml = getXML("testEXCEL_hyperlinks.xlsx").xml;
//external url
assertContains("<a href=\"http://tika.apache.org/\">", xml);
//mail url
assertContains("<a href=\"mailto:user@tika.apache.org?subject=help\">", xml);
//external linked file
assertContains("<a href=\"linked_file.txt.htm\">", xml);
//link on textbox
assertContains("<a href=\"http://tika.apache.org/1.12/gettingstarted.html\">", xml);
}
@Test
public void testEmbeddedPDFInPPTX() throws Exception {
List<Metadata> metadataList = getRecursiveMetadata("testPPT_EmbeddedPDF.pptx");
Metadata pdfMetadata1 = metadataList.get(4);
assertContains("Apache Tika", pdfMetadata1.get(RecursiveParserWrapper.TIKA_CONTENT));
Metadata pdfMetadata2 = metadataList.get(5);
assertContains("Hello World", pdfMetadata2.get(RecursiveParserWrapper.TIKA_CONTENT));
}
@Test
public void testEmbeddedPDFInXLSX() throws Exception {
List<Metadata> metadataList = getRecursiveMetadata("testExcel_embeddedPDF.xlsx");
Metadata pdfMetadata = metadataList.get(1);
assertContains("Hello World", pdfMetadata.get(RecursiveParserWrapper.TIKA_CONTENT));
}
@Test
public void testOrigSourcePath() throws Exception {
Metadata embed1_zip_metadata = getRecursiveMetadata("test_recursive_embedded.docx").get(11);
assertContains("C:\\Users\\tallison\\AppData\\Local\\Temp\\embed1.zip",
Arrays.asList(embed1_zip_metadata.getValues(TikaCoreProperties.ORIGINAL_RESOURCE_NAME)));
assertContains("C:\\Users\\tallison\\Desktop\\tmp\\New folder (2)\\embed1.zip",
Arrays.asList(embed1_zip_metadata.getValues(TikaCoreProperties.ORIGINAL_RESOURCE_NAME)));
}
@Test
public void testBigIntegersWGeneralFormat() throws Exception {
//TIKA-2025
String xml = getXML("testEXCEL_big_numbers.xlsx").xml;
assertContains("123456789012345", xml);//15 digit number
assertContains("123456789012346", xml);//15 digit formula
Locale locale = LocaleUtil.getUserLocale();
DecimalFormatSymbols symbols = new DecimalFormatSymbols(locale);
//16 digit number is treated as scientific notation as is the 16 digit formula
assertContains("1"+symbols.getDecimalSeparator()+"23456789012345E+15</td>\t"+
"<td>1"+symbols.getDecimalSeparator()+"23456789012345E+15", xml); }
@Test
public void testBoldHyperlink() throws Exception {
//TIKA-1255
String xml = getXML("testWORD_boldHyperlink.docx").xml;
xml = xml.replaceAll("\\s+", " ");
assertContains("<a href=\"http://tika.apache.org/\">hyper <b>link</b></a>", xml);
assertContains("<a href=\"http://tika.apache.org/\"><b>hyper</b> link</a>; bold", xml);
}
@Test
public void testLongForIntExceptionInSummaryDetails() throws Exception {
//TIKA-2055
assertContains("bold", getXML("testWORD_totalTimeOutOfRange.docx").xml);
}
@Test
public void testMacrosInDocm() throws Exception {
//test default is "don't extract macros"
for (Metadata metadata : getRecursiveMetadata("testWORD_macros.docm")) {
if (metadata.get(Metadata.CONTENT_TYPE).equals("text/x-vbasic")) {
fail("Shouldn't have extracted macros as default");
}
}
//now test that they were extracted
ParseContext context = new ParseContext();
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
officeParserConfig.setExtractMacros(true);
context.set(OfficeParserConfig.class, officeParserConfig);
Metadata minExpected = new Metadata();
minExpected.add(RecursiveParserWrapper.TIKA_CONTENT.getName(), "Sub Embolden()");
minExpected.add(RecursiveParserWrapper.TIKA_CONTENT.getName(), "Sub Italicize()");
minExpected.add(Metadata.CONTENT_TYPE, "text/x-vbasic");
minExpected.add(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE,
TikaCoreProperties.EmbeddedResourceType.MACRO.toString());
assertContainsAtLeast(minExpected, getRecursiveMetadata("testWORD_macros.docm", context));
//test configuring via config file
TikaConfig tikaConfig = new TikaConfig(this.getClass().getResourceAsStream("tika-config-dom-macros.xml"));
AutoDetectParser parser = new AutoDetectParser(tikaConfig);
assertContainsAtLeast(minExpected, getRecursiveMetadata("testWORD_macros.docm", parser));
}
@Test
public void testMacrosInPptm() throws Exception {
//test default is "don't extract macros"
for (Metadata metadata : getRecursiveMetadata("testPPT_macros.pptm")) {
if (metadata.get(Metadata.CONTENT_TYPE).equals("text/x-vbasic")) {
fail("Shouldn't have extracted macros as default");
}
}
//now test that they were extracted
ParseContext context = new ParseContext();
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
officeParserConfig.setExtractMacros(true);
context.set(OfficeParserConfig.class, officeParserConfig);
Metadata minExpected = new Metadata();
minExpected.add(RecursiveParserWrapper.TIKA_CONTENT.getName(), "Sub Embolden()");
minExpected.add(RecursiveParserWrapper.TIKA_CONTENT.getName(), "Sub Italicize()");
minExpected.add(Metadata.CONTENT_TYPE, "text/x-vbasic");
minExpected.add(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE,
TikaCoreProperties.EmbeddedResourceType.MACRO.toString());
assertContainsAtLeast(minExpected, getRecursiveMetadata("testPPT_macros.pptm", context));
//test configuring via config file
TikaConfig tikaConfig = new TikaConfig(this.getClass().getResourceAsStream("tika-config-dom-macros.xml"));
AutoDetectParser parser = new AutoDetectParser(tikaConfig);
assertContainsAtLeast(minExpected, getRecursiveMetadata("testPPT_macros.pptm", parser));
}
@Test
public void testMacroinXlsm() throws Exception {
//test default is "don't extract macros"
for (Metadata metadata : getRecursiveMetadata("testEXCEL_macro.xlsm")) {
if (metadata.get(Metadata.CONTENT_TYPE).equals("text/x-vbasic")) {
fail("Shouldn't have extracted macros as default");
}
}
//now test that they were extracted
ParseContext context = new ParseContext();
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
officeParserConfig.setExtractMacros(true);
context.set(OfficeParserConfig.class, officeParserConfig);
Metadata minExpected = new Metadata();
minExpected.add(RecursiveParserWrapper.TIKA_CONTENT.getName(), "Sub Dirty()");
minExpected.add(RecursiveParserWrapper.TIKA_CONTENT.getName(), "dirty dirt dirt");
minExpected.add(Metadata.CONTENT_TYPE, "text/x-vbasic");
minExpected.add(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE,
TikaCoreProperties.EmbeddedResourceType.MACRO.toString());
assertContainsAtLeast(minExpected,
getRecursiveMetadata("testEXCEL_macro.xlsm", context));
//test configuring via config file
TikaConfig tikaConfig = new TikaConfig(this.getClass().getResourceAsStream("tika-config-dom-macros.xml"));
AutoDetectParser parser = new AutoDetectParser(tikaConfig);
assertContainsAtLeast(minExpected, getRecursiveMetadata("testEXCEL_macro.xlsm", parser));
}
//@Test //use this for lightweight benchmarking to compare xwpf options
public void testBatch() throws Exception {
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
officeParserConfig.setUseSAXDocxExtractor(true);
long started = new Date().getTime();
int ex = 0;
for (int i = 0; i < 100; i++) {
for (File f : getResourceAsFile("/test-documents").listFiles()) {
if (!f.getName().endsWith(".docx")) {
continue;
}
try (InputStream is = TikaInputStream.get(f)) {
ParseContext parseContext = new ParseContext();
parseContext.set(OfficeParserConfig.class, officeParserConfig);
//test only the extraction of the main docx content, not embedded docs
parseContext.set(Parser.class, new EmptyParser());
Metadata metadata = new Metadata();
XMLResult r = getXML(is, parser, metadata, parseContext);
} catch (Exception e) {
ex++;
}
}
}
System.out.println("elapsed: "+(new Date().getTime()-started) + " with " + ex + " exceptions");
}
@Test
public void testInitializationViaConfig() throws Exception {
//NOTE: this test relies on a bug in the DOM extractor that
//is passing over the title information.
//once we fix that, this test will no longer be meaningful!
InputStream is = getClass().getResourceAsStream("/org/apache/tika/parser/microsoft/tika-config-sax-docx.xml");
assertNotNull(is);
TikaConfig tikaConfig = new TikaConfig(is);
AutoDetectParser p = new AutoDetectParser(tikaConfig);
XMLResult xml = getXML("testWORD_2006ml.docx", p, new Metadata());
assertContains("engaging title", xml.xml);
}
@Test
public void testExcelXLSB() throws Exception {
Detector detector = new DefaultDetector();
AutoDetectParser parser = new AutoDetectParser();
Metadata m = new Metadata();
m.add(Metadata.RESOURCE_NAME_KEY, "excel.xlsb");
// Should be detected correctly
MediaType type;
try (InputStream input = ExcelParserTest.class.getResourceAsStream(
"/test-documents/testEXCEL.xlsb")) {
type = detector.detect(input, m);
assertEquals("application/vnd.ms-excel.sheet.binary.macroenabled.12", type.toString());
}
// OfficeParser won't handle it
assertEquals(false, (new OfficeParser()).getSupportedTypes(new ParseContext()).contains(type));
// OOXMLParser will (soon) handle it
assertTrue((new OOXMLParser()).getSupportedTypes(new ParseContext()).contains(type));
// AutoDetectParser doesn't break on it
try (InputStream input = ExcelParserTest.class.getResourceAsStream("/test-documents/testEXCEL.xlsb")) {
ContentHandler handler = new BodyContentHandler(-1);
ParseContext context = new ParseContext();
context.set(Locale.class, Locale.US);
parser.parse(input, handler, m, context);
String content = handler.toString();
assertContains("This is an example spreadsheet", content);
}
}
@Test
public void testXLSBVarious() throws Exception {
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
officeParserConfig.setExtractMacros(true);
ParseContext parseContext = new ParseContext();
parseContext.set(OfficeParserConfig.class, officeParserConfig);
List<Metadata> metadataList = getRecursiveMetadata("testEXCEL_various.xlsb", parseContext);
assertEquals(4, metadataList.size());
String xml = metadataList.get(0).get(RecursiveParserWrapper.TIKA_CONTENT);
assertContains("<td>13</td>", xml);
assertContains("<td>13.1211231321</td>", xml);
assertContains("<td>$ 3.03</td>", xml);
assertContains("<td>20%</td>", xml);
assertContains("<td>13.12</td>", xml);
assertContains("<td>123456789012345</td>", xml);
assertContains("<td>1.23456789012345E+15</td>", xml);
assertContains("test comment2", xml);
assertContains("comment4 (end of row)", xml);
assertContains("<td>1/4</td>", xml);
assertContains("<td>3/9/17</td>", xml);
assertContains("<td>4</td>", xml);
assertContains("<td>2</td>", xml);
assertContains("<td> 46/1963</td>", xml);
assertContains("<td> 3/128</td>", xml);
assertContains("test textbox", xml);
assertContains("test WordArt", xml);
assertContains("<a href=\"http://lucene.apache.org/\">http://lucene.apache.org/</a>", xml);
assertContains("<a href=\"http://tika.apache.org/\">http://tika.apache.org/</a>", xml);
assertContains("OddLeftHeader OddCenterHeader OddRightHeader", xml);
assertContains("EvenLeftHeader EvenCenterHeader EvenRightHeader", xml);
assertContains("FirstPageLeftHeader FirstPageCenterHeader FirstPageRightHeader", xml);
assertContains("OddLeftFooter OddCenterFooter OddRightFooter", xml);
assertContains("EvenLeftFooter EvenCenterFooter EvenRightFooter", xml);
assertContains("FirstPageLeftFooter FirstPageCenterFooter FirstPageRightFooter", xml);
}
@Test
public void testPOI61034() throws Exception {
//tests temporary work around until POI 3.17-beta1 is released
XMLResult r = getXML("testEXCEL_poi-61034.xlsx");
Matcher m = Pattern.compile("<h1>(Sheet\\d+)</h1>").matcher(r.xml);
Set<String> seen = new HashSet<>();
while (m.find()) {
String sheetName = m.group(1);
if (seen.contains(sheetName)) {
fail("Should only see each sheet once: "+sheetName);
}
seen.add(sheetName);
}
}
}