/**
* Copyright 2015-2016 Abola Lee
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.github.abola.crawler;
import org.apache.commons.httpclient.Cookie;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.impl.SimpleLog;
import org.apache.commons.vfs2.CacheStrategy;
import org.apache.commons.vfs2.FileContent;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.FileSystemOptions;
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
import org.apache.commons.vfs2.provider.http.HttpFileSystemConfigBuilder;
import org.json.JSONArray;
import org.json.JSONObject;
import org.json.XML;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.parser.Parser;
import org.mozilla.universalchardet.UniversalDetector;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
* 資料爬蟲包
*
*
* @author Abola Lee <abola921@gmail.com>
*/
public class CrawlerPack {
static SimpleLog log = new SimpleLog("simple.logger.com.github.abola.crawler.CrawlerPack");
static StandardFileSystemManager fileSystem ;
static{
// create a Self-signed Server Certificates
// for pass SSL
XTrustProvider.install();
// Set default logging level "ERROR"
log.setLevel(SimpleLog.LOG_LEVEL_WARN);
try {
fileSystem = new StandardFileSystemManager();
fileSystem.setCacheStrategy(CacheStrategy.ON_CALL);
// change default logger to SimpleLog
fileSystem.setLogger(log);
fileSystem.init();
}catch(FileSystemException fse){
// ignore
}
}
static CrawlerPack defaultCrawler ;
/**
* Create a CrawlerPack instance
*
* @return CrawlerPack
*/
public static CrawlerPack start(){
if (null == defaultCrawler)
defaultCrawler = new CrawlerPack();
return defaultCrawler;
}
/**
* Setting global level logging
*
* example:
* CrawlerPack.setLoggerLevel( SimpleLog.LOG_LEVEL_INFO );
*
* @param level
*/
public static void setLoggerLevel(int level){
log.setLevel(level);
fileSystem.setLogger(log);
}
private String userAgent = "Mozilla/5.0 (CrawlerPack; )";
private List<Cookie> cookies = new ArrayList<>();
/**
* Creates a cookie with the given name and value.
*
* @param name the cookie name
* @param value the cookie value
* @return CrawlerPack
*/
public CrawlerPack addCookie(String name, String value){
if( null == name ) {
log.warn("addCookie: Cookie name null.");
return this;
}
cookies.add( new Cookie("", name, value) );
return this;
}
/**
* Creates a cookie with the given name, value, domain attribute,
* path attribute, expiration attribute, and secure attribute
*
* @param name the cookie name
* @param value the cookie value
* @param domain the domain this cookie can be sent to
* @param path the path prefix for which this cookie can be sent
* @param expires the {@link Date} at which this cookie expires,
* or <tt>null</tt> if the cookie expires at the end
* of the session
* @param secure if true this cookie can only be sent over secure
* connections
*
*/
public CrawlerPack addCookie(String domain, String name, String value,
String path, Date expires, boolean secure) {
if( null == name ) {
log.warn("addCookie: Cookie name null.");
return this;
}
cookies.add(new Cookie(domain, name, value, path, expires, secure));
return this;
}
/**
* Return a Cookie array
* and auto importing domain and path when domain was empty.
*
* @param uri required Apache Common VFS supported file systems and response JSON format content.
* @return Cookie[]
*/
Cookie[] getCookies(String uri){
if( null == cookies || 0 == cookies.size()) return null;
for(Cookie cookie: cookies){
if("".equals(cookie.getDomain())){
String domain = uri.replaceAll("^.*:\\/\\/([^\\/]+)[\\/]?.*$", "$1");
cookie.setDomain(domain);
cookie.setPath("/");
cookie.setExpiryDate(null);
cookie.setSecure(false);
}
}
return cookies.toArray(new Cookie[cookies.size()]);
}
/**
* Clear all cookies
*/
void clearCookies(){
log.trace("clearCookies: clear all cookies.");
cookies = new ArrayList<>();
}
/**
* 取得遠端格式為 JSON 的資料
*
* @param uri required Apache Common VFS supported file systems and response JSON format content.
* @return org.jsoup.nodes.Document
*/
public org.jsoup.nodes.Document getFromJson(String uri){
// 取回資料，並轉化為XML格式
String json = getFromRemote(uri);
// 將 json 轉化為 xml
String xml = jsonToXml(json);
// 轉化為 Jsoup 物件
return xmlToJsoupDoc(xml);
}
/**
* 取得遠端格式為 HTML/Html5 的資料
*
* @param uri required Apache Common VFS supported file systems and response HTML format content.
* @return org.jsoup.nodes.Document
*/
public org.jsoup.nodes.Document getFromHtml(String uri){
// 取回資料
String html = getFromRemote(uri);
// 轉化為 Jsoup 物件
return htmlToJsoupDoc(html);
}
/**
* 取得遠端格式為 XML 的資料
*
* @param uri required Apache Common VFS supported file systems and response XML format content.
* @return org.jsoup.nodes.Document
*/
public org.jsoup.nodes.Document getFromXml(String uri){
// 取回資料，並轉化為XML格式
String xml = getFromRemote(uri);
// 轉化為 Jsoup 物件
return xmlToJsoupDoc(xml);
}
/**
* 將 json 轉為 XML
*
* @param json a json format string.
* @return XML format string
*/
public String jsonToXml(String json){
String xml = "";
// 處理直接以陣列開頭的JSON，並指定給予 row 的 tag
if ( "[".equals( json.substring(0,1) ) ){
xml = XML.toString(new JSONArray(json), "row");
}else{
xml = XML.toString(new JSONObject(json));
}
return xml;
}
/**
* 透過 Apache Common VFS 套件 取回遠端的資源
*
* 能使用的協定參考：
* @see <a href="https://commons.apache.org/proper/commons-vfs/filesystems.html">commons-vfs filesystems</a>
*/
public String getFromRemote(String uri){
// clear cache
fileSystem.getFilesCache().close();
String remoteContent ;
String remoteEncoding = "utf-8";
log.debug("getFromRemote: Loading remote URI=" + uri);
FileContent fileContent ;
try {
FileSystemOptions fsOptions = new FileSystemOptions();
// set userAgent
HttpFileSystemConfigBuilder.getInstance().setUserAgent(fsOptions, userAgent);
// set cookie if cookies set
if (0 < this.cookies.size()) {
HttpFileSystemConfigBuilder.getInstance().setCookies(fsOptions, getCookies(uri));
}
log.debug("getFromRemote: userAgent=" + userAgent);
log.debug("getFromRemote: cookieSize=" + cookies.size());
log.debug("getFromRemote: cookies=" + cookies.toString());
fileContent = fileSystem.resolveFile(uri, fsOptions).getContent();
// 2016-03-22 only pure http/https auto detect encoding
if ("http".equalsIgnoreCase(uri.substring(0, 4))) {
fileContent.getSize(); // pass a bug {@link https://issues.apache.org/jira/browse/VFS-427}
remoteEncoding = fileContent.getContentInfo().getContentEncoding();
}
log.debug("getFromRemote: remoteEncoding=" + remoteEncoding + "(auto detect) ");
// 2016-03-21 修正zip file getContentEncoding 為null
if (null == remoteEncoding) remoteEncoding = "utf-8";
if (!"utf".equalsIgnoreCase(remoteEncoding.substring(0, 3))) {
log.debug("getFromRemote: remote content encoding=" + remoteEncoding);
// force charset encoding if setRemoteEncoding set
if (!"utf".equalsIgnoreCase(encoding.substring(0, 3))) {
remoteEncoding = encoding;
} else {
// auto detecting encoding
remoteEncoding = detectCharset(IOUtils.toByteArray(fileContent.getInputStream()));
log.debug("getFromRemote: real encoding=" + remoteEncoding);
}
}
// 透過 Apache VFS 取回指定的遠端資料
// 2016-02-29 fixed
remoteContent = IOUtils.toString(fileContent.getInputStream(), remoteEncoding);
} catch(FileSystemException fse){
log.warn("getFromRemote: FileSystemException=" + fse.getMessage());
remoteContent =null;
}catch(IOException ioe){
// return empty
log.warn("getFromRemote: IOException=" + ioe.getMessage());
remoteContent =null;
}catch(StringIndexOutOfBoundsException stre){
log.warn("getFromRemote: StringIndexOutOfBoundsException=" + stre.getMessage());
log.warn("getFromRemote: uri=" + uri );
log.warn(stre.getMessage());
remoteContent =null;
}
clearCookies();
log.debug("getFromRemote: remoteContent=\n" + remoteContent);
// any exception will return "null"
return remoteContent;
}
/**
* 將 HTML 轉化為 Jsoup Document 物件
*
* HTML的內容就使用Jsoup原生的 HTML Parser
*
* @param html Html document
* @return org.jsoup.nodes.Document
*/
public org.jsoup.nodes.Document htmlToJsoupDoc(String html){
// 將 html(html/html5) 轉為 jsoup Document 物件
Document jsoupDoc = Jsoup.parse(html, "UTF-8", Parser.htmlParser() );
jsoupDoc.charset(StandardCharsets.UTF_8);
return jsoupDoc;
}
// 替換字元：一定要是 a-zA-Z 開頭的組合
final static String prefix = "all-lower-case-prefix";
/**
* 將 XML 轉化為 Jsoup Document 物件
*
* Jsoup 1.9.1+ supported non-ascii tag
* -----
* 如果碰到Tag 名稱首字元非 a-zA-Z 的字元，jsoup 會解析為註解
* 所以必需用騙的先置入 prefix
* 再改寫xmlParse 在回傳時移除prefix
*
* @param xml XML format string
* @return org.jsoup.nodes.Document
*/
public org.jsoup.nodes.Document xmlToJsoupDoc(String xml){
// Tag 首字元非 a-zA-Z 時轉化為註解的問題
//xml = xml.replaceAll("<([^A-Za-z\\/! ][^\\/>]*)>", "<"+prefix.toLowerCase()+"$1>")
// .replaceAll("<\\/([^A-Za-z\\/ ][^\\/>]*)>", "</"+prefix.toLowerCase()+"$1>");
// 將 xml 轉為 jsoup Document 物件
//Document jsoupDoc = Jsoup.parse(xml, "", new Parser( new PrefixXmlTreeBuilder(prefix.toLowerCase()) ) );
Document jsoupDoc = Jsoup.parse(xml, "", Parser.xmlParser() );
jsoupDoc.charset(StandardCharsets.UTF_8);
return jsoupDoc;
}
private String encoding = "utf-8";
/**
* 指定來源資料的編碼格式
* 必需要在 get 前設定
*
* @return CrawlerPack
*/
public CrawlerPack setRemoteEncoding(String encoding){
log.debug("setRemoteEncoding: encoding=" + encoding);
this.encoding = encoding;
return this;
}
private String detectCharset(byte[] content){
log.debug("detectCharset: ");
return detectCharset(content, 0);
}
final Integer detectBuffer = 1000;
/**
* Detecting real content encoding
* @param content
* @param offset
* @return real charset encoding
*/
private String detectCharset(byte[] content, Integer offset){
log.debug("detectCharset: offset=" + offset);
// detect failed
if( offset > content.length ) return null;
UniversalDetector detector = new UniversalDetector(null);
detector.handleData(content, offset, content.length - offset > detectBuffer ? detectBuffer : content.length - offset);
detector.dataEnd();
String detectEncoding = detector.getDetectedCharset();
return null==detectEncoding?detectCharset(content,offset+detectBuffer):detectEncoding;
}
/**
* set header userAgent
*
* @param userAgent
* @return CrawlerPack
*/
public CrawlerPack setUserAgent(String userAgent){
log.debug("setUserAgent: userAgent=\"" + userAgent + "\"");
this.userAgent = userAgent;
return this;
}
}