/**
* License Agreement for OpenSearchServer
* <p/>
* Copyright (C) 2008-2016 Emmanuel Keller / Jaeksoft
* <p/>
* http://www.open-search-server.com
* <p/>
* This file is part of OpenSearchServer.
* <p/>
* OpenSearchServer is free software: you can redistribute it and/or
* modify it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* <p/>
* OpenSearchServer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* <p/>
* You should have received a copy of the GNU General Public License
* along with OpenSearchServer.
* If not, see <http://www.gnu.org/licenses/>.
**/
package com.jaeksoft.searchlib.crawler.web.database;
import com.jaeksoft.searchlib.ClientFactory;
import com.jaeksoft.searchlib.crawler.common.database.AbstractPropertyManager;
import com.jaeksoft.searchlib.crawler.web.spider.ProxyHandler;
import com.jaeksoft.searchlib.util.properties.PropertyItem;
import com.jaeksoft.searchlib.util.properties.PropertyItemListener;
import java.io.File;
import java.io.IOException;
public class WebPropertyManager extends AbstractPropertyManager implements PropertyItemListener {
final private PropertyItem<Integer> delayBetweenAccesses;
final private PropertyItem<Integer> fetchInterval;
final private PropertyItem<String> fetchIntervalUnit;
final private PropertyItem<Integer> maxUrlPerHost;
final private PropertyItem<Integer> maxUrlPerSession;
final private PropertyItem<String> userAgent;
final private PropertyItem<Integer> connectionTimeOut;
final private PropertyItem<Integer> maxDepth;
final private PropertyItem<Boolean> exclusionEnabled;
final private PropertyItem<Boolean> inclusionEnabled;
final private PropertyItem<Boolean> robotsTxtEnabled;
final private PropertyItem<Boolean> linkDetectionEnabled;
final private PropertyItem<String> screenshotBrowser;
final private PropertyItem<String> screenshotMethod;
final private PropertyItem<Integer> screenshotCaptureWidth;
final private PropertyItem<Integer> screenshotCaptureHeight;
final private PropertyItem<Integer> screenshotResizeWidth;
final private PropertyItem<Integer> screenshotResizeHeight;
final private PropertyItem<String> proxyHost;
final private PropertyItem<Integer> proxyPort;
final private PropertyItem<String> proxyExclusion;
final private PropertyItem<Boolean> proxyEnabled;
final private PropertyItem<String> proxyUsername;
final private PropertyItem<String> proxyPassword;
private ProxyHandler proxyHandler = null;
public WebPropertyManager(File file) throws IOException {
super(file, 1000);
delayBetweenAccesses = newIntegerProperty("delayBetweenAccesses", 10,
ClientFactory.INSTANCE.properties.getMinCrawlerDelay(), null);
fetchInterval = newIntegerProperty("fetchInterval", 30, 1, null);
fetchIntervalUnit = newStringProperty("fechIntervalUnit", "days");
maxUrlPerHost = newIntegerProperty("maxUrlPerHost", 100, 1, null);
maxUrlPerSession = newIntegerProperty("maxUrlPerSession", 10000, 1, null);
userAgent = newStringProperty("userAgent", "OpenSearchServer_Bot");
connectionTimeOut = newIntegerProperty("connectionTimeOut", 600, 0, null);
exclusionEnabled = newBooleanProperty("exclusionEnabled", true);
maxDepth = newIntegerProperty("maxDepth", null, null, null);
inclusionEnabled = newBooleanProperty("inclusionEnabled", true);
robotsTxtEnabled = newBooleanProperty("robotsTxtEnabled", true);
linkDetectionEnabled = newBooleanProperty("linkDetectionEnabled", true);
screenshotMethod = newStringProperty("screenshotMethod", "");
screenshotBrowser = newStringProperty("screenshotBrowser", "");
screenshotCaptureWidth = newIntegerProperty("screenshotCaptureWidth", 1024, 1, null);
screenshotCaptureHeight = newIntegerProperty("screenshotCaptureHeight", 768, 1, null);
screenshotResizeWidth = newIntegerProperty("screenshotResizeWidth", 240, 1, null);
screenshotResizeHeight = newIntegerProperty("screenshotResizeHeight", 180, 1, null);
proxyHost = newStringProperty("proxyHost", "");
proxyPort = newIntegerProperty("proxyPort", 8080, null, null);
proxyExclusion = newStringProperty("proxyExclusion", "");
proxyEnabled = newBooleanProperty("proxyEnabled", false);
proxyUsername = newStringProperty("proxyUsername", "");
proxyPassword = newStringProperty("proxyPassword", "");
proxyHost.addListener(this);
proxyPort.addListener(this);
proxyExclusion.addListener(this);
proxyEnabled.addListener(this);
proxyUsername.addListener(this);
proxyPassword.addListener(this);
}
public PropertyItem<String> getProxyHost() {
return proxyHost;
}
public PropertyItem<String> getProxyExclusion() {
return proxyExclusion;
}
public PropertyItem<Boolean> getProxyEnabled() {
return proxyEnabled;
}
public PropertyItem<Integer> getProxyPort() {
return proxyPort;
}
public PropertyItem<String> getProxyUsername() {
return proxyUsername;
}
public PropertyItem<String> getProxyPassword() {
return proxyPassword;
}
public PropertyItem<Integer> getFetchInterval() {
return fetchInterval;
}
public PropertyItem<Integer> getMaxUrlPerHost() {
return maxUrlPerHost;
}
public PropertyItem<Integer> getMaxUrlPerSession() {
return maxUrlPerSession;
}
public PropertyItem<String> getUserAgent() {
return userAgent;
}
public PropertyItem<Integer> getConnectionTimeOut() {
return connectionTimeOut;
}
public PropertyItem<Integer> getMaxDepth() {
return maxDepth;
}
public PropertyItem<String> getFetchIntervalUnit() {
return fetchIntervalUnit;
}
public PropertyItem<Boolean> getInclusionEnabled() {
return inclusionEnabled;
}
public PropertyItem<Boolean> getExclusionEnabled() {
return exclusionEnabled;
}
public PropertyItem<Integer> getDelayBetweenAccesses() {
return delayBetweenAccesses;
}
public PropertyItem<Boolean> getRobotsTxtEnabled() {
return robotsTxtEnabled;
}
public PropertyItem<Boolean> getLinkDetectionEnabled() {
return linkDetectionEnabled;
}
public PropertyItem<String> getScreenshotMethod() {
return screenshotMethod;
}
public PropertyItem<String> getScreenshotBrowser() {
return screenshotBrowser;
}
public PropertyItem<Integer> getScreenshotCaptureWidth() {
return screenshotCaptureWidth;
}
public PropertyItem<Integer> getScreenshotCaptureHeight() {
return screenshotCaptureHeight;
}
public PropertyItem<Integer> getScreenshotResizeWidth() {
return screenshotResizeWidth;
}
public PropertyItem<Integer> getScreenshotResizeHeight() {
return screenshotResizeHeight;
}
@Override
public void hasBeenSet(PropertyItem<?> prop) {
synchronized (this) {
if (prop == proxyHost || prop == proxyPort || prop == proxyExclusion || prop == proxyUsername
|| prop == proxyPassword)
proxyHandler = null;
}
}
public ProxyHandler getProxyHandler() throws IOException {
synchronized (this) {
if (!proxyEnabled.getValue())
return null;
if (proxyHandler != null)
return proxyHandler;
proxyHandler = new ProxyHandler(this);
return proxyHandler;
}
}
}