/*
* Created by Angel Leon (@gubatron), Alden Torres (aldenml)
* Copyright (c) 2011-2014, FrostWire(R). All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.frostwire.search;
import java.io.IOException;
import java.util.Map;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.frostwire.search.domainalias.DomainAliasManager;
import com.frostwire.util.HttpClient;
import com.frostwire.util.HttpClientFactory;
import com.frostwire.util.URLUtils;
import com.frostwire.util.UserAgentGenerator;
/**
*
* @author gubatron
* @author aldenml
*
*/
public abstract class WebSearchPerformer extends AbstractSearchPerformer {
private static final Logger LOG = LoggerFactory.getLogger(WebSearchPerformer.class);
private static final String DEFAULT_USER_AGENT = UserAgentGenerator.getUserAgent();
private static final String[] STREAMABLE_EXTENSIONS = new String[] { "mp3", "ogg", "wma", "wmv", "m4a", "aac", "flac", "mp4", "flv", "mov", "mpg", "mpeg", "3gp", "m4v", "webm" };
private final String keywords;
private final String encodedKeywords;
private final int timeout;
private final HttpClient client;
private final DomainAliasManager domainAliasManager;
public WebSearchPerformer(DomainAliasManager domainAliasManager, long token, String keywords, int timeout) {
super(token);
if (domainAliasManager == null) {
throw new IllegalArgumentException("domainAliasManager can't be null");
}
this.domainAliasManager = domainAliasManager;
this.keywords = keywords;
this.encodedKeywords = URLUtils.encode(keywords);
this.timeout = timeout;
this.client = HttpClientFactory.newDefaultInstance();
}
public final String getKeywords() {
return keywords;
}
public final String getEncodedKeywords() {
return encodedKeywords;
}
@Override
public void crawl(CrawlableSearchResult sr) {
LOG.warn("Review your logic, calling deep search without implementation for: " + sr);
}
/**
* Allow to perform the HTTP operation using the same internal http client.
*
* @param url
* @return the web page (html)
*/
public String fetch(String url) throws IOException {
return fetch(url, null, null);
}
public String fetch(String url, String cookie, Map<String, String> customHeaders) throws IOException {
return client.get(url, timeout, DEFAULT_USER_AGENT, null, cookie, customHeaders);
}
public String post(String url, Map<String, String> formData) {
return client.post(url, timeout, DEFAULT_USER_AGENT, formData);
}
/**
* Allow to perform the HTTP operation using the same internal http client.
*
* @param url
* @return the raw bytes from the http connection
*/
public final byte[] fetchBytes(String url) {
return fetchBytes(url, null, timeout);
}
protected final byte[] fetchBytes(String url, String referrer, int timeout) {
if (url.startsWith("htt")) { // http(s)
return client.getBytes(url, timeout, DEFAULT_USER_AGENT, referrer);
} else {
return null;
}
}
protected final boolean isStreamable(String filename) {
String ext = FilenameUtils.getExtension(filename);
for (String s : STREAMABLE_EXTENSIONS) {
if (s.equals(ext)) {
return true; // fast return
}
}
return false;
}
public String getDomainNameToUse() {
return domainAliasManager.getDomainNameToUse();
}
public String getDefaultDomainName() {
return domainAliasManager.getDefaultDomain();
}
public DomainAliasManager getDomainAliasManager() {
return domainAliasManager;
}
/**
* The current domain has failed, mark it offline and let's try check if other mirrors are alive.
*/
protected final void checkAccesibleDomains() {
LOG.debug("WebSearchPerformer.checkAccesibleDomains()! " + getDefaultDomainName() + " Performer failed, marking " + getDomainNameToUse() + " offline, checking domains.");
domainAliasManager.markDomainOffline(getDomainNameToUse());
domainAliasManager.checkStatuses(this);
}
}