// Jericho HTML Parser - Java based library for analysing and manipulating HTML
// Version 3.2
// Copyright (C) 2004-2009 Martin Jericho
// http://jericho.htmlparser.net/
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of either one of the following licences:
//
// 1. The Eclipse Public License (EPL) version 1.0,
// included in this distribution in the file licence-epl-1.0.html
// or available at http://www.eclipse.org/legal/epl-v10.html
//
// 2. The GNU Lesser General Public License (LGPL) version 2.1 or later,
// included in this distribution in the file licence-lgpl-2.1.txt
// or available at http://www.gnu.org/licenses/lgpl.txt
//
// This library is distributed on an "AS IS" basis,
// WITHOUT WARRANTY OF ANY KIND, either express or implied.
// See the individual licence texts for more details.
package net.htmlparser.jericho;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.io.Closeable;
import java.io.Reader;
import java.io.Writer;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.CharBuffer;
import java.nio.BufferOverflowException;
import java.net.URL;
import java.net.URLConnection;
/**
* Represents a streamed source HTML document.
* <p>
* This class provides a means, via the {@link #iterator()} method, of sequentially parsing every {@linkplain Tag tag}, {@linkplain CharacterReference character reference}
* and <a href="#PlainText">plain text</a> segment contained within the source document using a minimum amount of memory.
* <p>
* In contrast, the standard {@link Source} class stores the entire source text in memory and caches every tag parsed,
* resulting in memory problems when attempting to parse very large files.
* <p>
* The {@link #iterator() iterator} parses and returns each segment as the source text is streamed in.
* Previous segments are discarded for garbage collection.
* Source documents up to 2GB in size can be processed, a limit which is imposed by the java language because of its use of the <code>int</code> data type to index string operations.
* <p>
* There is however a significant trade-off in functionality when using the <code>StreamedSource</code> class as opposed to the {@link Source} class.
* The {@link Tag#getElement()} method is not supported on tags that are returned by the iterator, nor are any methods that use the {@link Element} class in any way.
* The {@link Segment#getSource()} method is also not supported.
* <p>
* Most of the methods and constructors in this class mirror similarly named methods in the {@link Source} class where the same functionality is available.
* <p>
* See the description of the {@link #iterator()} method for a typical usage example of this class.
* <p>
* In contrast to a {@link Source} object, the <code>Reader</code> or <code>InputStream</code> specified in the constructor or created implicitly by the constructor
* remains open for the life of the <code>StreamedSource</code> object. If the stream is created internally, it is automatically {@linkplain #close() closed}
* when the end of the stream is reached or the <code>StreamedSource</code> object is {@linkplain #finalize() finalized}.
* However a <code>Reader</code> or <code>InputStream</code> that is specified directly in a constructor is never closed automatically, as it can not be assumed
* that the application has no further use for it. It is the user's responsibility to ensure it is closed in this case.
* Explicitly calling the {@link #close()} method on the <code>StreamedSource</code> object ensures that all resources used by it are closed, regardless of whether
* they were created internally or supplied externally.
* <p>
* The functionality provided by <code>StreamedSource</code> is similar to a <a target="_blank" href="http://en.wikipedia.org/wiki/StAX">StAX</a> parser,
* but with some important benefits:
* <ul>
* <li>
* The source document does not have to be valid XML. It can be plain HTML, can contain invalid syntax, undefined entities,
* incorrectly nested elements, {@linkplain TagType#isServerTag() server tags}, or anything else that is commonly found in
* "<a target="_blank" href="http://en.wikipedia.org/wiki/Tag_soup">tag soup</a>".
* <li>
* Every single syntactical construct in the source document's original text is included in the iterator, including the
* {@linkplain StartTagType#XML_DECLARATION XML declaration}, {@linkplain CharacterReference character references}, {@linkplain StartTagType#COMMENT comments},
* {@linkplain StartTagType#CDATA_SECTION CDATA sections} and {@linkplain TagType#isServerTag() server tags},
* each providing the segment's {@linkplain Segment#getBegin() begin} and {@linkplain Segment#getEnd() end} position in the source document.
* This allows an exact copy of the original document to be generated, allowing modifications to be made only where they are explicitly required.
* This is not possible with either <a target="_blank" href="http://en.wikipedia.org/wiki/Simple_API_for_XML">SAX</a> or
* <a target="_blank" href="http://en.wikipedia.org/wiki/StAX">StAX</a>, which to some extent provide interpretations of the content of the XML
* instead of the syntactial structures used in the original source document.
* </ul>
* <p>
* The following table summarises the differences between the <code>StreamedSource</code>, StAX and SAX interfaces.
* Note that some of the available features are documented as optional and may not be supported by all implementations of StAX and SAX.
* <p>
* <style type="text/css">
* table#ParserComparison td, table#ParserComparison th {padding: 0px 5px 0px 5px}
* .checkmark {text-align: center; font-size: 12pt}
* </style>
* <table id="ParserComparison" class="bordered" cellspacing="0">
* <tr><th class="LabelColumn">Feature</th><th><code>StreamedSource</code></th><th><a target="_blank" href="http://en.wikipedia.org/wiki/StAX">StAX</a></th><th><a target="_blank" href="http://en.wikipedia.org/wiki/Simple_API_for_XML">SAX</a></th></tr>
* <tr><td class="LabelColumn">Parse XML</td><td class="checkmark">●</td><td class="checkmark">●</td><td class="checkmark">●</td></tr>
* <tr><td class="LabelColumn">Parse entities without DTD</td><td class="checkmark">●</td><td class="checkmark"></td><td class="checkmark"></td></tr>
* <tr><td class="LabelColumn">Automatically validate XML</td><td class="checkmark"></td><td class="checkmark">●</td><td class="checkmark">●</td></tr>
* <tr><td class="LabelColumn">Parse HTML</td><td class="checkmark">●</td><td class="checkmark"></td><td class="checkmark"></td></tr>
* <tr><td class="LabelColumn">Tolerant of syntax or nesting errors</td><td class="checkmark">●</td><td class="checkmark"></td><td class="checkmark"></td></tr>
* <tr><td class="LabelColumn">Provide begin and end character positions of each event<sup>1</sup></td><td class="checkmark">●</td><td class="checkmark">○</td><td class="checkmark"></td></tr>
* <tr><td class="LabelColumn">Provide source text of each event</td><td class="checkmark">●</td><td class="checkmark"></td><td class="checkmark"></td></tr>
* <tr><td class="LabelColumn">Handle {@linkplain TagType#isServerTag() server tag} events</td><td class="checkmark">●</td><td class="checkmark"></td><td class="checkmark"></td></tr>
* <tr><td class="LabelColumn">Handle {@linkplain StartTagType#XML_DECLARATION XML declaration} event</td><td class="checkmark">●</td><td class="checkmark"></td><td class="checkmark"></td></tr>
* <tr><td class="LabelColumn">Handle {@linkplain StartTagType#COMMENT comment} events</td><td class="checkmark">●</td><td class="checkmark">●</td><td class="checkmark">●</td></tr>
* <tr><td class="LabelColumn">Handle {@linkplain StartTagType#CDATA_SECTION CDATA section} events</td><td class="checkmark">●</td><td class="checkmark">●</td><td class="checkmark">●</td></tr>
* <tr><td class="LabelColumn">Handle {@linkplain StartTagType#DOCTYPE_DECLARATION document type declaration} event</td><td class="checkmark">●</td><td class="checkmark">●</td><td class="checkmark">●</td></tr>
* <tr><td class="LabelColumn">Handle {@linkplain CharacterReference character reference} events</td><td class="checkmark">●</td><td class="checkmark"></td><td class="checkmark"></td></tr>
* <tr><td class="LabelColumn">Allow chunking of plain text</td><td class="checkmark">●</td><td class="checkmark">●</td><td class="checkmark">●</td></tr>
* <tr><td class="LabelColumn">Allow chunking of comment text</td><td class="checkmark"></td><td class="checkmark"></td><td class="checkmark"></td></tr>
* <tr><td class="LabelColumn">Allow chunking of CDATA section text</td><td class="checkmark"></td><td class="checkmark"></td><td class="checkmark">●</td></tr>
* <tr><td class="LabelColumn">Allow specification of maximum buffer size</td><td class="checkmark">●</td><td class="checkmark"></td><td class="checkmark"></td></tr>
* </table>
* <sup>1</sup> StAX optionally reports the "offset" of each event but this could be either byte or character position depending on the source.
* <p>
* Note that the {@link OutputDocument} class can not be used to create a modified version of a streamed source document.
* Instead, the output document must be constructed manually from the segments provided by the {@link #iterator() iterator}.
* <p>
* <code>StreamedSource</code> objects are not thread safe.
*/
public final class StreamedSource implements Iterable<Segment>, Closeable {
private final StreamedText streamedText;
private final StreamedParseText streamedParseText;
private final Source source;
private final Closeable closeable; // internally created closeable object should be cleaned up internally.
private final boolean automaticClose;
private boolean coalescing=false;
private boolean handleTags=true;
private Config.UnterminatedCharacterReferenceSettings unterminatedCharacterReferenceSettings=Config.CurrentCompatibilityMode.getUnterminatedCharacterReferenceSettings(false);
private boolean isInitialised=false;
private Segment currentSegment=null;
private Segment nextParsedSegment=START_SEGMENT;
private boolean isXML;
private static final boolean assumeNoNestedTags=false;
private static final Segment START_SEGMENT=new Segment(-1,-1);
private StreamedSource(final Reader reader, final boolean automaticClose, final String encoding, final String encodingSpecificationInfo, final String preliminaryEncodingInfo) throws IOException {
closeable=reader;
this.automaticClose=automaticClose;
streamedText=new StreamedText(reader);
streamedParseText=new StreamedParseText(streamedText);
source=new Source(streamedText,streamedParseText,encoding,encodingSpecificationInfo,preliminaryEncodingInfo);
}
private StreamedSource(final EncodingDetector encodingDetector, final boolean automaticClose) throws IOException {
this(encodingDetector.openReader(),automaticClose,encodingDetector.getEncoding(),encodingDetector.getEncodingSpecificationInfo(),encodingDetector.getPreliminaryEncoding()+": "+encodingDetector.getPreliminaryEncodingSpecificationInfo());
}
/**
* Constructs a new <code>StreamedSource</code> object by loading the content from the specified <code>Reader</code>.
* <p>
* If the specified reader is an instance of <code>InputStreamReader</code>, the {@link #getEncoding()} method of the
* created <code>StreamedSource</code> object returns the encoding from <code>InputStreamReader.getEncoding()</code>.
*
* @param reader the <code>java.io.Reader</code> from which to load the source text.
* @throws java.io.IOException if an I/O error occurs.
*/
public StreamedSource(final Reader reader) throws IOException {
this(reader,false,(reader instanceof InputStreamReader) ? ((InputStreamReader)reader).getEncoding() : null,(reader instanceof InputStreamReader) ? "InputStreamReader.getEncoding() of constructor argument" : null,null);
}
/**
* Constructs a new <code>StreamedSource</code> object by loading the content from the specified <code>InputStream</code>.
* <p>
* The algorithm for detecting the character {@linkplain #getEncoding() encoding} of the source document from the raw bytes
* of the specified input stream is the same as that for the {@link Source#Source(URLConnection) Source(URLConnection)} constructor of the {@link Source} class,
* except that the first step is not possible as there is no
* <a target="_blank" href="http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17">Content-Type</a> header to check.
* <p>
* If the specified <code>InputStream</code> does not support the <code>mark</code> method, the algorithm that determines the encoding may have to wrap it
* in a <code>BufferedInputStream</code> in order to look ahead at the encoding meta data.
* This extra layer of buffering will then remain in place for the life of the <code>StreamedSource</code>, possibly impacting memory usage and/or degrading performance.
* It is always preferable to use the {@link #StreamedSource(Reader)} constructor if the encoding is known in advance.
*
* @param inputStream the <code>java.io.InputStream</code> from which to load the source text.
* @throws java.io.IOException if an I/O error occurs.
* @see #getEncoding()
*/
public StreamedSource(final InputStream inputStream) throws IOException {
this(new EncodingDetector(inputStream),false);
}
/**
* Constructs a new <code>StreamedSource</code> object by loading the content from the specified URL.
* <p>
* This is equivalent to {@link #StreamedSource(URLConnection) StreamedSource(url.openConnection())}.
*
* @param url the URL from which to load the source text.
* @throws java.io.IOException if an I/O error occurs.
* @see #getEncoding()
*/
public StreamedSource(final URL url) throws IOException {
this(new EncodingDetector(url.openConnection()),true);
}
/**
* Constructs a new <code>StreamedSource</code> object by loading the content from the specified <code>URLConnection</code>.
* <p>
* The algorithm for detecting the character {@linkplain #getEncoding() encoding} of the source document is identical to that described in the
* {@link Source#Source(URLConnection) Source(URLConnection)} constructor of the {@link Source} class.
* <p>
* The algorithm that determines the encoding may have to wrap the input stream in a <code>BufferedInputStream</code> in order to look ahead
* at the encoding meta data if the encoding is not specified in the HTTP headers.
* This extra layer of buffering will then remain in place for the life of the <code>StreamedSource</code>, possibly impacting memory usage and/or degrading performance.
* It is always preferable to use the {@link #StreamedSource(Reader)} constructor if the encoding is known in advance.
*
* @param urlConnection the URL connection from which to load the source text.
* @throws java.io.IOException if an I/O error occurs.
* @see #getEncoding()
*/
public StreamedSource(final URLConnection urlConnection) throws IOException {
this(new EncodingDetector(urlConnection),true);
}
/**
* Constructs a new <code>StreamedSource</code> object from the specified text.
* <p>
* Although the <code>CharSequence</code> argument of this constructor apparently contradicts the notion of streaming in the source text,
* it can still benefits over the equivalent use of the standard {@link Source} class.
* <p>
* Firstly, using the <code>StreamedSource</code> class to iterate the nodes of an in-memory <code>CharSequence</code> source document still requires much less memory
* than the equivalent operation using the standard {@link Source} class.
* <p>
* Secondly, the specified <code>CharSequence</code> object could possibly implement its own paging mechanism to minimise memory usage.
* <p>
* If the specified <code>CharSequence</code> is mutable, its state must not be modified while the <code>StreamedSource</code> is in use.
*
* @param text the source text.
*/
public StreamedSource(final CharSequence text) {
closeable=null;
automaticClose=false;
streamedText=new StreamedText(text);
streamedParseText=new StreamedParseText(streamedText);
source=new Source(text,streamedParseText,null,"Document specified encoding can not be determined automatically from a streamed source",null);
}
/**
* Specifies an existing character array to use for buffering the incoming character stream.
* <p>
* The specified buffer is fixed for the life of the <code>StreamedSource</code> object, in contrast to the default buffer which can be automatically replaced
* by a larger buffer as needed.
* This means that if a {@linkplain Tag tag} (including a {@linkplain StartTagType#COMMENT comment} or {@linkplain StartTagType#CDATA_SECTION CDATA section}) is
* encountered that is larger than the specified buffer, an unrecoverable <code>BufferOverflowException</code> is thrown.
* This exception is also thrown if {@link #setCoalescing(boolean) coalescing} has been enabled and a plain text segment is encountered
* that is larger than the specified buffer.
* <p>
* In general this method should only be used if there needs to be an absolute maximum memory limit imposed on the parser, where that requirement is more important
* than the ability to parse any source document successfully.
* <p>
* This method can only be called before the {@link #iterator()} method has been called.
*
* @param buffer an existing character array to use for buffering the incoming character stream, must not be <code>null</code>.
* @return this <code>StreamedSource</code> instance, allowing multiple property setting methods to be chained in a single statement.
* @throws IllegalStateException if the {@link #iterator()} method has already been called.
*/
public StreamedSource setBuffer(char[] buffer) {
if (isInitialised) throw new IllegalStateException("setBuffer() can only be called before iterator() is called");
streamedText.setBuffer(buffer);
return this;
}
/**
* Specifies whether an unbroken section of <a href="#PlainText">plain text</a> in the source document should always be coalesced into a single {@link Segment} by the {@linkplain #iterator() iterator}.
* <p>
* If this property is set to the <b>default</b> value of <code>false</code>,
* and a section of plain text is encountered in the document that is larger than the current {@linkplain #getBufferSize() buffer size},
* the text is <i>chunked</i> into multiple consecutive plain text segments in order to minimise memory usage.
* <p>
* If this property is set to <code>true</code> then chunking is disabled, ensuring that consecutive plain text segments are never generated,
* but instead forcing the internal buffer to expand to fit the largest section of plain text.
* <p>
* Note that {@link CharacterReference} segments are always handled separately from plain text, regardless of the value of this property.
* For this reason, algorithms that process element content almost always have to be designed to expect the text in multiple segments
* in order to handle character references, so there is usually no advantage in {@linkplain #setCoalescing(boolean) coalescing} plain text segments.
*
* @param coalescing the new value of the coalescing property.
* @return this <code>StreamedSource</code> instance, allowing multiple property setting methods to be chained in a single statement.
* @throws IllegalStateException if the {@link #iterator()} method has already been called.
*/
public StreamedSource setCoalescing(final boolean coalescing) {
if (isInitialised) throw new IllegalStateException("setPlainTextWriter() can only be called before iterator() is called");
this.coalescing=coalescing;
return this;
}
/**
* Closes the underlying <code>Reader</code> or <code>InputStream</code> and releases any system resources associated with it.
* <p>
* If the stream is already closed then invoking this method has no effect.
*
* @throws IOException if an I/O error occurs.
*/
public void close() throws IOException {
if (closeable!=null) closeable.close();
}
/**
* Returns the character encoding scheme of the source byte stream used to create this object.
* <p>
* This method works in essentially the same way as the {@link Source#getEncoding()} method.
* <p>
* If the byte stream used to create this object does not support the <code>mark</code> method, the algorithm that determines the encoding may have to wrap it
* in a <code>BufferedInputStream</code> in order to look ahead at the encoding meta data.
* This extra layer of buffering will then remain in place for the life of the <code>StreamedSource</code>, possibly impacting memory usage and/or degrading performance.
* It is always preferable to use the {@link #StreamedSource(Reader)} constructor if the encoding is known in advance.
* <p>
* The {@link #getEncodingSpecificationInfo()} method returns a simple description of how the value of this method was determined.
*
* @return the character encoding scheme of the source byte stream used to create this object, or <code>null</code> if the encoding is not known.
* @see #getEncodingSpecificationInfo()
*/
public String getEncoding() {
return source.getEncoding();
}
/**
* Returns a concise description of how the {@linkplain #getEncoding() encoding} of the source document was determined.
* <p>
* The description is intended for informational purposes only.
* It is not guaranteed to have any particular format and can not be reliably parsed.
*
* @return a concise description of how the {@linkplain #getEncoding() encoding} of the source document was determined.
* @see #getEncoding()
*/
public String getEncodingSpecificationInfo() {
return source.getEncodingSpecificationInfo();
}
/**
* Returns the preliminary encoding of the source document together with a concise description of how it was determined.
* <p>
* This method works in essentially the same way as the {@link Source#getPreliminaryEncodingInfo()} method.
* <p>
* The description returned by this method is intended for informational purposes only.
* It is not guaranteed to have any particular format and can not be reliably parsed.
*
* @return the preliminary encoding of the source document together with a concise description of how it was determined, or <code>null</code> if no preliminary encoding was required.
* @see #getEncoding()
*/
public String getPreliminaryEncodingInfo() {
return source.getPreliminaryEncodingInfo();
}
/**
* Returns an iterator over every {@linkplain Tag tag}, {@linkplain CharacterReference character reference} and plain text segment contained within the source document.
* <p>
* <i><a name="PlainText">Plain text</a></i> is defined as all text that is not part of a {@link Tag} or {@link CharacterReference}.
* <p>
* This results in a sequential walk-through of the entire source document.
* The {@linkplain Segment#getEnd() end} position of each segment should correspond with the {@linkplain Segment#getBegin() begin} position of the subsequent segment,
* unless any of the tags are enclosed by other tags.
* This could happen if there are {@linkplain TagType#isServerTag() server tags} present in the document, or in rare circumstances where the
* {@linkplain StartTagType#DOCTYPE_DECLARATION document type declaration} contains {@linkplain StartTagType#MARKUP_DECLARATION markup declarations}.
* <p>
* Each segment generated by the iterator is parsed as the source text is streamed in. Previous segments are discarded for garbage collection.
* <p>
* If a section of plain text is encountered in the document that is larger than the current {@linkplain #getBufferSize() buffer size},
* the text is <i>chunked</i> into multiple consecutive plain text segments in order to minimise memory usage.
* Setting the {@link #setCoalescing(boolean) Coalescing} property to <code>true</code> disables chunking, ensuring that consecutive plain text segments are never generated,
* but instead forcing the internal buffer to expand to fit the largest section of plain text.
* Note that {@link CharacterReference} segments are always handled separately from plain text, regardless of whether {@linkplain #setCoalescing(boolean) coalescing}
* is enabled. For this reason, algorithms that process element content almost always have to be designed to expect the text in multiple segments
* in order to handle character references, so there is usually no advantage in {@linkplain #setCoalescing(boolean) coalescing} plain text segments.
* <p>
* Character references that are found inside tags, such as those present inside attribute values, do not generate separate segments from the iterator.
* <p>
* This method may only be called once on any particular <code>StreamedSource</code> instance.
* <p>
* <dl>
* <dt>Example:</dt>
* <dd>
* <p>
* The following code demonstrates the typical (implied) usage of this method through the <code>Iterable</code> interface
* to make an exact copy of the document from <code>reader</code> to <code>writer</code> (assuming no server tags are present):
* </p>
* <pre>
* StreamedSource streamedSource=new StreamedSource(reader);
* for (Segment segment : streamedSource) {
* if (segment instanceof Tag) {
* Tag tag=(Tag)segment;
* // HANDLE TAG
* // Uncomment the following line to ensure each tag is valid XML:
* // writer.write(tag.tidy()); continue;
* } else if (segment instanceof CharacterReference) {
* CharacterReference characterReference=(CharacterReference)segment;
* // HANDLE CHARACTER REFERENCE
* // Uncomment the following line to decode all character references instead of copying them verbatim:
* // characterReference.appendCharTo(writer); continue;
* } else {
* // HANDLE PLAIN TEXT
* }
* // unless specific handling has prevented getting to here, simply output the segment as is:
* writer.write(segment.toString());
* }</pre>
* <p>Note that the last line <code>writer.write(segment.toString())</code> in the above code can be replaced with the following for improved performance:</p>
* <pre>
* CharBuffer charBuffer=streamedSource.getCurrentSegmentCharBuffer();
* writer.write(charBuffer.array(),charBuffer.position(),charBuffer.length());</pre>
* </dd>
* <dd>
* <p>
* The following code demonstrates how to process the plain text content of a specific element, in this case to print the content of every paragraph element:
* </p>
* <pre>
* StreamedSource streamedSource=new StreamedSource(reader);
* StringBuilder sb=new StringBuilder();
* boolean insideParagraphElement=false;
* for (Segment segment : streamedSource) {
* if (segment instanceof Tag) {
* Tag tag=(Tag)segment;
* if (tag.getName().equals("p")) {
* if (tag instanceof StartTag) {
* insideParagraphElement=true;
* sb.setLength(0);
* } else { // tag instanceof EndTag
* insideParagraphElement=false;
* System.out.println(sb.toString());
* }
* }
* } else if (insideParagraphElement) {
* if (segment instanceof CharacterReference) {
* ((CharacterReference)segment).appendCharTo(sb);
* } else {
* sb.append(segment);
* }
* }
* }</pre>
* </dd>
* </dl>
* @return an iterator over every {@linkplain Tag tag}, {@linkplain CharacterReference character reference} and plain text segment contained within the source document.
*/
public Iterator<Segment> iterator() {
if (isInitialised) throw new IllegalStateException("iterator() can only be called once");
isInitialised=true;
return new StreamedSourceIterator();
}
/**
* Returns the current {@link Segment} from the {@linkplain #iterator()}.
* <p>
* This is defined as the last {@link Segment} returned from the iterator's <code>next()</code> method.
* <p>
* This method returns <code>null</code> if the iterator's <code>next()</code> method has never been called, or its
* <code>hasNext()</code> method has returned the value <code>false</code>.
*
* @return the current {@link Segment} from the {@linkplain #iterator()}.
*/
public Segment getCurrentSegment() {
return currentSegment;
}
/**
* Returns a <code>CharBuffer</code> containing the source text of the {@linkplain #getCurrentSegment() current segment}.
* <p>
* The returned <code>CharBuffer</code> provides a window into the internal <code>char[]</code> buffer including the position and length that spans the
* {@linkplain #getCurrentSegment() current segment}.
* <p>
* For example, the following code writes the source text of the current segment to <code>writer</code>:
* <p>
* <code>CharBuffer charBuffer=streamedSource.getCurrentSegmentCharBuffer();</code><br />
* <code>writer.write(charBuffer.array(),charBuffer.position(),charBuffer.length());</code>
* <p>
* This may provide a performance benefit over the standard way of accessing the source text of the current segment,
* which is to use the <code>CharSequence</code> interface of the segment directly, or to call {@link Segment#toString()}.
* <p>
* Because this <code>CharBuffer</code> is a direct window into the internal buffer of the <code>StreamedSource</code>, the contents of the
* <code>CharBuffer.array()</code> must not be modified, and the array is only guaranteed to hold the segment source text until the
* iterator's <code>hasNext()</code> or <code>next()</code> method is next called.
*
* @return a <code>CharBuffer</code> containing the source text of the {@linkplain #getCurrentSegment() current segment}.
*/
public CharBuffer getCurrentSegmentCharBuffer() {
return streamedText.getCharBuffer(currentSegment.getBegin(),currentSegment.end);
}
/**
* Indicates whether the source document is likely to be <a target="_blank" href="http://www.w3.org/TR/REC-xml/">XML</a>.
* <p>
* The algorithm used to determine this is designed to be relatively inexpensive and to provide an accurate result in
* most normal situations.
* An exact determination of whether the source document is XML would require a much more complex analysis of the text.
* <p>
* The algorithm is as follows:
* <ol class="HalfSeparated">
* <li>If the document begins with an {@linkplain StartTagType#XML_DECLARATION XML declaration}, it is an XML document.
* <li>If the document begins with a {@linkplain StartTagType#DOCTYPE_DECLARATION document type declaration} that contains the text
* "<code>xhtml</code>", it is an <a target="_blank" href="http://www.w3.org/TR/xhtml1/">XHTML</a> document, and hence
* also an XML document.
* <li>If none of the above conditions are met, assume the document is normal HTML, and therefore not an XML document.
* </ol>
* <p>
* This method can only be called after the {@link #iterator()} method has been called.
*
* @return <code>true</code> if the source document is likely to be <a target="_blank" href="http://www.w3.org/TR/REC-xml/">XML</a>, otherwise <code>false</code>.
* @throws IllegalStateException if the {@link #iterator()} method has not yet been called.
*/
public boolean isXML() {
if (!isInitialised) throw new IllegalStateException("isXML() method only available after iterator() has been called");
return isXML;
}
/**
* Sets the {@link Logger} that handles log messages.
* <p>
* Specifying a <code>null</code> argument disables logging completely for operations performed on this <code>StreamedSource</code> object.
* <p>
* A logger instance is created automatically for each <code>StreamedSource</code> object in the same way as is described in the
* {@link Source#setLogger(Logger)} method.
*
* @param logger the logger that will handle log messages, or <code>null</code> to disable logging.
* @see Config#LoggerProvider
*/
public void setLogger(final Logger logger) {
source.setLogger(logger);
}
/**
* Returns the {@link Logger} that handles log messages.
* <p>
* A logger instance is created automatically for each <code>StreamedSource</code> object using the {@link LoggerProvider}
* specified by the static {@link Config#LoggerProvider} property.
* This can be overridden by calling the {@link #setLogger(Logger)} method.
* The name used for all automatically created logger instances is "<code>net.htmlparser.jericho</code>".
*
* @return the {@link Logger} that handles log messages, or <code>null</code> if logging is disabled.
*/
public Logger getLogger() {
return source.getLogger();
}
/**
* Returns the current size of the internal character buffer.
* <p>
* This information is generally useful only for investigating memory and performance issues.
*
* @return the current size of the internal character buffer.
*/
public int getBufferSize() {
return streamedText.getBuffer().length;
}
/**
* Returns a string representation of the object as generated by the default <code>Object.toString()</code> implementation.
* <p>
* In contrast to the {@link Source#toString()} implementation, it is generally not possible for this method to return the entire source text.
*
* @return a string representation of the object as generated by the default <code>Object.toString()</code> implementation.
*/
public String toString() {
return super.toString();
}
/**
* Called by the garbage collector on an object when garbage collection determines that there are no more references to the object.
* <p>
* This implementation calls the {@link #close()} method if the underlying <code>Reader</code> or <code>InputStream</code> stream was created internally.
*/
protected void finalize() {
automaticClose();
}
StreamedSource setHandleTags(final boolean handleTags) {
this.handleTags=handleTags;
return this;
}
StreamedSource setUnterminatedCharacterReferenceSettings(final Config.UnterminatedCharacterReferenceSettings unterminatedCharacterReferenceSettings) {
this.unterminatedCharacterReferenceSettings=unterminatedCharacterReferenceSettings;
return this;
}
StreamedSource setSearchBegin(final int begin) {
if (isInitialised) throw new IllegalStateException("setSearchBegin() can only be called before iterator() is called");
final int segmentEnd=begin-1;
nextParsedSegment=new Segment(segmentEnd,segmentEnd);
return this;
}
private void automaticClose() {
if (automaticClose) try {close();} catch (IOException ex) {}
}
private static boolean isXML(final Segment firstNonTextSegment) {
if (firstNonTextSegment==null || !(firstNonTextSegment instanceof Tag)) return false;
Tag tag=(Tag)firstNonTextSegment;
if (tag.getTagType()==StartTagType.XML_DECLARATION) return true;
// if document has a DOCTYPE declaration and it contains the text "xhtml", it is an XML document:
if (tag.source.getParseText().indexOf("xhtml",tag.begin,tag.end)!=-1) return true;
return false;
}
private class StreamedSourceIterator implements Iterator<Segment> {
private final boolean coalescing;
private final boolean handleTags;
private Config.UnterminatedCharacterReferenceSettings unterminatedCharacterReferenceSettings;
private Segment nextSegment;
private int plainTextSegmentBegin=0;
private final char[] charByRef=new char[1]; // used to pass a single character by reference
public StreamedSourceIterator() {
coalescing=StreamedSource.this.coalescing;
handleTags=StreamedSource.this.handleTags;
unterminatedCharacterReferenceSettings=StreamedSource.this.unterminatedCharacterReferenceSettings;
loadNextParsedSegment();
isXML=isXML(nextParsedSegment);
}
public boolean hasNext() {
if (nextSegment==Tag.NOT_CACHED) loadNextParsedSegment();
return nextSegment!=null;
}
public Segment next() {
if (!hasNext()) throw new NoSuchElementException();
final Segment result=nextSegment;
nextSegment=(result==nextParsedSegment) ? Tag.NOT_CACHED : nextParsedSegment;
streamedText.setMinRequiredBufferBegin(result.end); // guaranteed not to be discarded until next call to loadNextParsedSegment()
currentSegment=result;
return result;
}
public void remove() {
throw new UnsupportedOperationException();
}
private final void loadNextParsedSegment() {
nextParsedSegment=findNextParsedSegment();
final int plainTextSegmentEnd=(nextParsedSegment!=null) ? nextParsedSegment.begin : streamedText.length();
nextSegment=(plainTextSegmentBegin<plainTextSegmentEnd) ? new Segment(source,plainTextSegmentBegin,plainTextSegmentEnd) : nextParsedSegment;
if (nextParsedSegment!=null && plainTextSegmentBegin<nextParsedSegment.end) plainTextSegmentBegin=nextParsedSegment.end;
}
private final Segment findNextParsedSegment() {
try {
int i=(nextParsedSegment instanceof StartTag && ((StartTag)nextParsedSegment).getTagType()==StartTagType.SERVER_COMMON_COMMENT)
? nextParsedSegment.getEnd()
: nextParsedSegment.getBegin()+1;
final int searchEnd=coalescing ? streamedText.getEnd() : streamedText.getBufferOverflowPosition();
while (i<searchEnd) {
final char ch=streamedText.charAt(i);
if (ch=='&') {
if (i>=source.fullSequentialParseData[0]) { // do not handle character references inside tags or script elements
final CharacterReference characterReference=CharacterReference.construct(source,i,unterminatedCharacterReferenceSettings);
if (characterReference!=null) return characterReference;
}
} else if (handleTags && ch=='<') {
final Tag tag=TagType.getTagAt(source,i,false,assumeNoNestedTags);
if (tag!=null && !tag.isUnregistered()) {
final TagType tagType=tag.getTagType();
if (tag.end>source.fullSequentialParseData[0] && tagType!=StartTagType.DOCTYPE_DECLARATION) {
source.fullSequentialParseData[0]=(tagType==StartTagType.NORMAL && tag.name==HTMLElementName.SCRIPT && !((StartTag)tag).isEmptyElementTag()) ? Integer.MAX_VALUE : tag.end;
}
return tag;
}
}
i++;
}
if (i<streamedText.getEnd()) {
// not coalescing, reached buffer overflow position
return new Segment(source,plainTextSegmentBegin,i);
}
} catch (BufferOverflowException ex) {
// Unrecoverable buffer overflow - close the reader if it was created internally:
automaticClose();
throw ex;
} catch (IndexOutOfBoundsException ex) {
// normal way to catch end of stream.
}
// streamedText.length() is now guaranteed to return document length
// End of stream has been reached, can close the reader if it was created internally:
automaticClose();
return null;
}
}
}