/**
* Copyright 2016 benjobs
* <p>
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.opencron.common.utils;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Map;
import java.util.Random;
/**
* @author benjobs 2017-05
*/
public class HttpClientUtils {
private static PoolingHttpClientConnectionManager connectionManager;
private static String EMPTY_STR = "";
private static String UTF_8 = "UTF-8";
private static void init() {
if (connectionManager == null) {
connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(50);// 整个连接池最大连接数
connectionManager.setDefaultMaxPerRoute(5);// 每路由最大连接数，默认值是2
}
}
/**
* 通过连接池获取HttpClient
*
* @return
*/
private static CloseableHttpClient getHttpClient() {
init();
return HttpClients.custom().setConnectionManager(connectionManager).build();
}
private static Random random = new Random();
private static String[] userAgents = null;
/**
* 获取http模拟请求User-Agent信息
*
* @param n
* @return
*/
public static String getUserAgent(int n) {
if (userAgents == null) {
userAgents = new String[24];
userAgents[0] = "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"; // easou
userAgents[1] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36";
userAgents[2] = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MDDR; .NET4.0C; .NET4.0E; .NET CLR 1.1.4322; Tablet PC 2.0)"; // 360spider
userAgents[3] = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
userAgents[4] = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)";
userAgents[5] = "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 BIDUBrowser/2.x Safari/537.31";
userAgents[6] = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Win64; x64; Trident/4.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Tablet PC 2.0)";
userAgents[7] = "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1";
userAgents[8] = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 2.0.50727; .NET CLR 1.1.4322)";
userAgents[9] = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)";
userAgents[10] = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET4.0E)";
userAgents[11] = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; EasyBits GO v1.0; InfoPath.1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)";
userAgents[12] = "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1";
userAgents[13] = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0)";
userAgents[14] = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/7.0; MASP)";
userAgents[15] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.802.30 Safari/535.1 SE 2.X MetaSr 1.0";
userAgents[16] = "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD";
userAgents[17] = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36";
userAgents[18] = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31";
userAgents[19] = "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider";
userAgents[20] = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36";
userAgents[21] = "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/3.5 Safari/536.11";
userAgents[22] = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0 AlexaToolbar/alxf-2.19";
userAgents[23] = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
}
if (n > userAgents.length) {
n = 0;
}
return userAgents[n];
}
/**
* @param url
* @return
*/
public static String httpGetRequest(String url) {
HttpGet httpGet = new HttpGet(url);
return getResult(httpGet);
}
public static String httpGetRequest(String url, Map<String, Object> params) throws URISyntaxException {
URIBuilder ub = new URIBuilder();
ub.setPath(url);
ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
ub.setParameters(pairs);
HttpGet httpGet = new HttpGet(ub.build());
return getResult(httpGet);
}
public static String httpGetRequest(String url, Map<String, Object> headers, Map<String, Object> params)
throws URISyntaxException {
URIBuilder ub = new URIBuilder();
ub.setPath(url);
ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
ub.setParameters(pairs);
HttpGet httpGet = new HttpGet(ub.build());
for (Map.Entry<String, Object> param : headers.entrySet()) {
httpGet.addHeader(param.getKey(), String.valueOf(param.getValue()));
}
return getResult(httpGet);
}
public static String httpPostRequest(String url) {
HttpPost httpPost = new HttpPost(url);
return getResult(httpPost);
}
public static String httpPostRequest(String url, Map<String, Object> params) throws UnsupportedEncodingException {
HttpPost httpPost = new HttpPost(url);
ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
httpPost.setEntity(new UrlEncodedFormEntity(pairs, UTF_8));
return getResult(httpPost);
}
public static String httpPostRequest(String url, String params) throws UnsupportedEncodingException {
HttpPost httpPost = new HttpPost(url);
StringEntity entitystring = new StringEntity(params, "utf-8");//解决中文乱码问题
entitystring.setContentEncoding("UTF-8");
entitystring.setContentType("application/json");
httpPost.setEntity(entitystring);
return getResult(httpPost);
}
public static String httpPostRequest(String url, Map<String, Object> headers, Map<String, Object> params)
throws UnsupportedEncodingException {
HttpPost httpPost = new HttpPost(url);
for (Map.Entry<String, Object> param : headers.entrySet()) {
httpPost.addHeader(param.getKey(), String.valueOf(param.getValue()));
}
ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
httpPost.setEntity(new UrlEncodedFormEntity(pairs, UTF_8));
return getResult(httpPost);
}
private static ArrayList<NameValuePair> covertParams2NVPS(Map<String, Object> params) {
ArrayList<NameValuePair> pairs = new ArrayList<NameValuePair>();
for (Map.Entry<String, Object> param : params.entrySet()) {
pairs.add(new BasicNameValuePair(param.getKey(), String.valueOf(param.getValue())));
}
return pairs;
}
/**
* 处理Http请求
*
* @param request
* @return
*/
private static String getResult(HttpRequestBase request) {
CloseableHttpClient httpClient = getHttpClient();
try {
CloseableHttpResponse response = httpClient.execute(request);
// response.getStatusLine().getStatusCode();
HttpEntity entity = response.getEntity();
if (entity != null) {
// long len = entity.getContentLength();// -1 表示长度未知
String result = EntityUtils.toString(entity);
response.close();
// httpClient.close();
return result;
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
}
return EMPTY_STR;
}
}