java爬取数据时候动态获取分页数据

当我们在java代码里请求分页网址的时候需要对分页数据动态获取，写固定数值的话不能灵活，
所以需要先获取总数后在去进行分页

import com.alibaba.fastjson.JSONObject;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.web.client.ResponseErrorHandler;
import org.springframework.web.client.RestTemplate;
import java.io.IOException;

public class ClientHttpUtil {

	
	public static RestTemplate getRest() {
		RestTemplate rest = new RestTemplate();
		rest.setErrorHandler(new ResponseErrorHandler() {
			@Override
			public boolean hasError(ClientHttpResponse clientHttpResponse) throws IOException {
				return true;
			}

			@Override
			public void handleError(ClientHttpResponse clientHttpResponse) throws IOException {
			}
		});
		return rest;
	}

	
	public static ResponseEntity getresponseEntity(RestTemplate rest, String url, HttpMethod request, HttpHeaders headers) {
		return rest.exchange(url, request, new HttpEntity(headers), String.class);
	}

	
	public static ResponseEntity getresponseEntityByReplacePage(RestTemplate rest, String url, Integer pageNo, Integer pageSize, HttpMethod request, HttpHeaders headers) {
		if (url.contains("pageNo=") || url.contains("pageSize=")) {
			url = url.replace("pageNo=", "pageNo=" + pageNo);
			url = url.replace("pageSize=", "pageSize=" + pageSize);
		}
		return rest.exchange(url, request, new HttpEntity(headers), String.class);
	}

	
	public static Integer getPage(ResponseEntity responseEntity, Integer size) {
		if (responseEntity.getStatusCodeValue() == 200) {
			String detail = responseEntity.getBody();
			JSONObject detailObject = JSONObject.parseObject(detail);
			int total = Integer.parseInt(detailObject.get("total").toString());
			return total % size == 0 ? total / size : total / size + 1;
		}
		return null;
	}
}

package org.springblade.modules.api.controller;

import org.apache.http.client.utils.HttpClientUtils;
import org.springblade.modules.api.utils.ClientHttpUtil;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;


public class Test {
	public static void main(String[] args) {
		String url = "http://localhost:8080/pageTest?pageNo=&pageSize=";
		// 获取连接对象
		RestTemplate rest = ClientHttpUtil.getRest();
		// 设值请求头部
		HttpHeaders headers = new HttpHeaders();
		// 请求类型
		headers.add("content-type", "application/x-www-form-urlencoded;charset=utf-8");
		// 账号的token数据，没有headers可以为null
		headers.add("token", "");
		// 每页显示的页数，自定义
		Integer size = 3000;
		// getPage获取页数的方法 , getresponseEntityByReplacePage 请求网址的方法动态分页用
		Integer page = ClientHttpUtil.getPage(ClientHttpUtil.getresponseEntityByReplacePage(rest, url, 1, 5, HttpMethod.GET, headers), size);
		// 获取页数后遍历取得所有值
		for (Integer i = 1; i < page + 1; i++) {
			ResponseEntity entity = ClientHttpUtil.getresponseEntityByReplacePage(rest, url, i, size, HttpMethod.GET, headers);
			// 之后解析该数据就可以啦
		}
	}
	
	public static JSONObject getJSONObjectByResponseEntity(ResponseEntity responseEntity) {
		if (responseEntity.getStatusCodeValue() == 200) {
			String body = responseEntity.getBody();
			JSONObject object = JSONObject.parseObject(body);
			if (object.getInteger("code") == 200) {
				return object;
			}
		}
		return null;
	}
}