栏目分类:
子分类:
返回
名师互学网用户登录
快速导航关闭
当前搜索
当前分类
子分类
实用工具
热门搜索
名师互学网 > IT > 软件开发 > 后端开发 > Java

Httpclient5 多线程爬取页面

Java 更新时间: 发布时间: IT归档 最新发布 模块sitemap 名妆网 法律咨询 聚返吧 英语巴士网 伯小乐 网商动力

Httpclient5 多线程爬取页面

添加maven依赖

        
            org.apache.httpcomponents.client5
            httpclient5
            5.1.3
        

编写抓取工具类

package com.s.c.crawler.mock.words.httpclient;

import org.apache.hc.client5.http.HttpRequestRetryStrategy;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hc.core5.net.URIBuilder;

import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
import java.util.*;
import java.util.Map.Entry;

public class HttpTools {
 
    public static String KEY_STATUS_CODE = "statusCode";
    public static String KEY_CONTENT = "content";

    
    private final static PoolingHttpClientConnectionManager poolConnManager = new PoolingHttpClientConnectionManager();  //连接池管理器
    private final static HttpRequestRetryStrategy httpRequestRetryStrategy = DefaultHttpRequestRetryStrategy.INSTANCE; //重试策略

    static {   //类加载的时候 设置最大连接数 和 每个路由的最大连接数
        poolConnManager.setMaxTotal(2000); 
        poolConnManager.setDefaultMaxPerRoute(1000);
    }
 
 
    
    private static CloseableHttpClient getCloseableHttpClient() {
        CloseableHttpClient httpClient = HttpClients.custom()
                .setConnectionManager(poolConnManager)
                .setRetryStrategy(httpRequestRetryStrategy)
                .build();
 
        return httpClient;
    }
 
    
    private static Map buildResultMap(CloseableHttpResponse response, HttpEntity entity) throws
            IOException, ParseException {
        Map result;
        result = new HashMap<>(2);
        result.put(KEY_STATUS_CODE, response.getCode());  //status code
        if (entity != null) {
            result.put(KEY_CONTENT, EntityUtils.toString(entity, "UTF-8")); //message content
        }
        return result;
    }
 
    public static String postRequest(String httpUrl, Map args) {
		
        CloseableHttpClient httpClient = getCloseableHttpClient();
        HttpPost httpPost = new HttpPost(httpUrl);
        CloseableHttpResponse response = null;
        try {
 
        	//4、声明存放参数的List集合
            List params = new ArrayList();
            Set> entrySet = args.entrySet();
			for (Entry entry : entrySet) {
				if(entry.getValue()!=null) {
					params.add(new BasicNameValuePair(entry.getKey(), String.valueOf(entry.getValue())));
				}else {
					params.add(new BasicNameValuePair(entry.getKey(), null));
				}
			}
			//5、创建form表单对象
            UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params, Charset.forName("utf-8"));
            //6、把表单对象设置到httpPost中
            httpPost.setEntity(formEntity);
            
            response = httpClient.execute(httpPost);
            HttpEntity entity = response.getEntity();
            String responseBody = null;
			if (entity != null) {
				responseBody = EntityUtils.toString(entity, "UTF-8");
				//关闭流
				EntityUtils.consumeQuietly(entity);
			}
			return responseBody;
 
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        } finally {
            if (response != null) {
                try {
                    EntityUtils.consume(response.getEntity());
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
		
	}
    
    
    public static Map postJson(String url, String message) {
 
        Map result = null;
        CloseableHttpClient httpClient = getCloseableHttpClient();
        HttpPost httpPost = new HttpPost(url);
        CloseableHttpResponse response = null;
        try {
 
            httpPost.setHeader("Accept", "application/json;charset=UTF-8");
            httpPost.setHeader("Content-Type", "application/json");
 
            StringEntity stringEntity = new StringEntity(message, ContentType.create("application/json;charset=UTF-8"));

            httpPost.setEntity(stringEntity);
            response = httpClient.execute(httpPost);
            HttpEntity entity = response.getEntity();
            result = buildResultMap(response, entity);
 
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (response != null) {
                try {
                    EntityUtils.consume(response.getEntity());
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return result;
    }

    public static String getRequest(String httpUrl, Map args) {

        CloseableHttpClient httpClient = getCloseableHttpClient();
        HttpGet httpGet = new HttpGet(httpUrl);

        CloseableHttpResponse response = null;
        try {
            if (args != null && !args.isEmpty()) {
                URIBuilder uriBuilder = new URIBuilder(httpGet.getUri());
                Set> entrySet = args.entrySet();
                for (Entry entry : entrySet) {
                    if(entry.getValue()!=null) {
                        uriBuilder.addParameter(entry.getKey(), String.valueOf(entry.getValue()));
                    }
                }
                URI uri = uriBuilder.build();
                httpGet.setUri(uri);
            }
            response = httpClient.execute(httpGet);
            HttpEntity entity = response.getEntity();
            String responseBody = null;
            if (entity != null) {
                responseBody = EntityUtils.toString(entity, "UTF-8");
                //关闭流
                EntityUtils.consumeQuietly(entity);
            }
            return responseBody;

        } catch (Exception e) {
            e.printStackTrace();
            return null;
        } finally {
            if (response != null) {
                try {
                    EntityUtils.consume(response.getEntity());
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
    public static void main(String[] args) {
        Map map = new HashMap<>();
        map.put("q", "北京");
        String result = HttpTools.getRequest("https://cn.bing.com/search", map);
        System.out.println(result);
    }
}

转载请注明:文章转载自 www.mshxw.com
本文地址:https://www.mshxw.com/it/962059.html
我们一直用心在做
关于我们 文章归档 网站地图 联系我们

版权所有 (c)2021-2022 MSHXW.COM

ICP备案号:晋ICP备2021003244-6号