java 实现通过 post 方式提交json参数操作

由于所爬取的网站需要验证码，通过网页的开发人员工具【F12】及在线http post,get接口测试请求工具(http://coolaf.com/)发现访问时加上请求头header 信息时可以跳过验证码校验。

而且该网站只接受post请求，对提交的参数也只接受json格式，否则请求失败。

现将通过 post 方式提交json参数的方法记录如下：

import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestbase;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;


public class PostJsonParamsTest {

  // 超时时间
  private static final int RUN_TIME =10000;

  // 爬取初始页数
  private String page;

  public static void main(String[] args) throws Exception {
    PostJsonParamsTest crawl = new PostJsonParamsTest();

    // 请求的url地址
    String url ="http://www.gzcredit.gov.cn/Service/CreditService.asmx/searchOrgWithPage";
    // 设置起始访问页码
    crawl.setPage("1");
    String isStop = "";

    // 设置请求
    HttpRequestbase request = null;
    request = new HttpPost(url);

    try {
      // 设置config
      RequestConfig requestConfig = RequestConfig.custom()
     .setSocketTimeout(RUN_TIME)
     .setConnectTimeout(RUN_TIME)
     .setConnectionRequestTimeout(RUN_TIME)
     .build();
      request.setConfig(requestConfig);

      // json 格式的 post 参数
      String postParams ="{"condition":{"qymc":"%%%%","cydw":""},"pageNo":"+crawl.getPage()+","pageSize":100,count:2709846}";
      System.out.println(postParams);
      HttpEntity httpEntity = new StringEntity(postParams);
      ((HttpPost) request).setEntity(httpEntity);

      // 添加请求头，可以绕过验证码
      request.addHeader("Accept","application/json, text/javascript, *
  private String crawlList(HttpRequestbase request){
    int statusCode = 0;

    // 下面两种方式都可以用来创建客户端连接,相当于打开了一个浏览器
    CloseableHttpClient httpClient = HttpClients.createDefault(); 
    // HttpClient httpClient = HttpClientBuilder.create().build();

    HttpEntity httpEntity = null;
    HttpResponse response = null;
    try {      
      try { 
 response = httpClient.execute(request);
      } catch (Exception e){
 e.printStackTrace();
 EntityUtils.consumeQuietly(httpEntity);
 return "重跑";
      } 

      //打印状态
      statusCode =response.getStatusLine().getStatusCode();
      if(statusCode!=200){
 EntityUtils.consumeQuietly(httpEntity);
 return "重跑";
      }
      //实体
      httpEntity = response.getEntity();
      String searchListStr = EntityUtils.toString(httpEntity,"GBK").replaceAll("\\米", "米");
      String allData = (String) JSONObject.parseObject(searchListStr).get("d");
      // 字符串值中间含双引号的替换处理
      String s = allData.replaceAll("\{"","{'")
   .replaceAll("":"", "':'")
   .replaceAll("","", "','")
   .replaceAll("":", "':")
   .replaceAll(","", ",'")
   .replaceAll(""\}", "'}")
   .replaceAll(""", "")
   .replaceAll("'", """)
   .replaceAll("
", "") 
   .replaceAll("t", "")
   .replaceAll("\\", "?");
      JSonObject jsonData = JSONObject.parseObject(s);
      JSonArray jsonContent = jsonData.getJSonArray("orgList");

      searchListStr = null;
      allData = null; 
      s = null;

      if (jsonContent==null || jsonContent.size()<1) {
 return "重跑";
      }
      System.out.println(jsonContent.toJSonString());
      return "爬取";
    } catch (Exception e) {
      e.printStackTrace();
      return "重跑";
    } finally{
      EntityUtils.consumeQuietly(httpEntity);
    }
  }

  private String getPage() {
    return page;
  }

  private void setPage(String page) {
    this.page = page;
  }

}

补充知识：JAVA利用HttpClient发送post请求，将请求数据放到body里

我就废话不多说了，大家还是直接看代码吧~

  
  public static String doPostBodyData(String url, String bodyData) throws Exception{
    String result = "";
    CloseableHttpClient httpClient = null;
    CloseableHttpResponse response = null;
    try {
      HttpPost httpPost = getHttpPost(url, null); // 请求地址
      httpPost.setEntity(new StringEntity(bodyData, Encoding));
      httpClient = getHttpClient();
      // 得到返回的response
      response = httpClient.execute(httpPost);
      HttpEntity entity = response.getEntity();
      result = getResult(entity, Encoding);
    } catch (Exception e) {
      throw e;
    } finally {
      // 关闭httpClient
      if (null != httpClient) {
 httpClient.close();
      }
      // 关闭response
      if (null != response) {
 EntityUtils.consume(response.getEntity()); // 会自动释放连接
 response.close();
      }
    }
    return result;
  }

以上这篇java 实现通过 post 方式提交json参数操作就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持考高分网。