添加maven依赖
org.apache.httpcomponents.client5 httpclient55.1.3
编写抓取工具类
package com.s.c.crawler.mock.words.httpclient;
import org.apache.hc.client5.http.HttpRequestRetryStrategy;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
import java.util.*;
import java.util.Map.Entry;
public class HttpTools {
public static String KEY_STATUS_CODE = "statusCode";
public static String KEY_CONTENT = "content";
private final static PoolingHttpClientConnectionManager poolConnManager = new PoolingHttpClientConnectionManager(); //连接池管理器
private final static HttpRequestRetryStrategy httpRequestRetryStrategy = DefaultHttpRequestRetryStrategy.INSTANCE; //重试策略
static { //类加载的时候 设置最大连接数 和 每个路由的最大连接数
poolConnManager.setMaxTotal(2000);
poolConnManager.setDefaultMaxPerRoute(1000);
}
private static CloseableHttpClient getCloseableHttpClient() {
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(poolConnManager)
.setRetryStrategy(httpRequestRetryStrategy)
.build();
return httpClient;
}
private static Map buildResultMap(CloseableHttpResponse response, HttpEntity entity) throws
IOException, ParseException {
Map result;
result = new HashMap<>(2);
result.put(KEY_STATUS_CODE, response.getCode()); //status code
if (entity != null) {
result.put(KEY_CONTENT, EntityUtils.toString(entity, "UTF-8")); //message content
}
return result;
}
public static String postRequest(String httpUrl, Map args) {
CloseableHttpClient httpClient = getCloseableHttpClient();
HttpPost httpPost = new HttpPost(httpUrl);
CloseableHttpResponse response = null;
try {
//4、声明存放参数的List集合
List params = new ArrayList();
Set> entrySet = args.entrySet();
for (Entry entry : entrySet) {
if(entry.getValue()!=null) {
params.add(new BasicNameValuePair(entry.getKey(), String.valueOf(entry.getValue())));
}else {
params.add(new BasicNameValuePair(entry.getKey(), null));
}
}
//5、创建form表单对象
UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params, Charset.forName("utf-8"));
//6、把表单对象设置到httpPost中
httpPost.setEntity(formEntity);
response = httpClient.execute(httpPost);
HttpEntity entity = response.getEntity();
String responseBody = null;
if (entity != null) {
responseBody = EntityUtils.toString(entity, "UTF-8");
//关闭流
EntityUtils.consumeQuietly(entity);
}
return responseBody;
} catch (Exception e) {
e.printStackTrace();
return null;
} finally {
if (response != null) {
try {
EntityUtils.consume(response.getEntity());
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
public static Map postJson(String url, String message) {
Map result = null;
CloseableHttpClient httpClient = getCloseableHttpClient();
HttpPost httpPost = new HttpPost(url);
CloseableHttpResponse response = null;
try {
httpPost.setHeader("Accept", "application/json;charset=UTF-8");
httpPost.setHeader("Content-Type", "application/json");
StringEntity stringEntity = new StringEntity(message, ContentType.create("application/json;charset=UTF-8"));
httpPost.setEntity(stringEntity);
response = httpClient.execute(httpPost);
HttpEntity entity = response.getEntity();
result = buildResultMap(response, entity);
} catch (Exception e) {
e.printStackTrace();
} finally {
if (response != null) {
try {
EntityUtils.consume(response.getEntity());
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result;
}
public static String getRequest(String httpUrl, Map args) {
CloseableHttpClient httpClient = getCloseableHttpClient();
HttpGet httpGet = new HttpGet(httpUrl);
CloseableHttpResponse response = null;
try {
if (args != null && !args.isEmpty()) {
URIBuilder uriBuilder = new URIBuilder(httpGet.getUri());
Set> entrySet = args.entrySet();
for (Entry entry : entrySet) {
if(entry.getValue()!=null) {
uriBuilder.addParameter(entry.getKey(), String.valueOf(entry.getValue()));
}
}
URI uri = uriBuilder.build();
httpGet.setUri(uri);
}
response = httpClient.execute(httpGet);
HttpEntity entity = response.getEntity();
String responseBody = null;
if (entity != null) {
responseBody = EntityUtils.toString(entity, "UTF-8");
//关闭流
EntityUtils.consumeQuietly(entity);
}
return responseBody;
} catch (Exception e) {
e.printStackTrace();
return null;
} finally {
if (response != null) {
try {
EntityUtils.consume(response.getEntity());
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
public static void main(String[] args) {
Map map = new HashMap<>();
map.put("q", "北京");
String result = HttpTools.getRequest("https://cn.bing.com/search", map);
System.out.println(result);
}
}



