java实现登录之后抓取数据

最近做了一个从网络上抓取数据的一个小程序。主要关于信贷方面，收集的一些黑名单网站，从该网站上抓取到自己系统中。

也找了一些资料，觉得没有一个很好的，全面的例子。因此在这里做个笔记提醒自己。

首先需要一个jsoup的jar包，我用的1.6.0。。下载地址为：http://pan.baidu.com/s/1mgqOuHa

1，获取网页内容（核心代码，技术有限没封装）。

2，登录之后抓取网页数据（如何在请求中携带cookie）。

3，获取网站的ajax请求方法（返回json）。

以上这三点我就用一个类全部包含（比较糙望见谅，直接copy代码过去，应该就可以用）

一，这个类分别有这上面的1,2,3三中方法，直接main方法可以进行测试

package com.minxinloan.black.web.utils;

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.StringTokenizer;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;

import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Jsoup;
import org.jsoup.nodes.document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class cookieUtil {

  public final static String CONTENT_TYPE = "Content-Type";

  public static void main(String[] args) {
    
    //String loginURL = "http://www.p2peye.com/member.php?mod=logging&action=login&loginsubmit=yes&loginhash=Lsc66&username=puqiuxiaomao&password=a1234567";
    String listURL = "http://www.p2peye.com/blacklist.php?p=2";
    String logURL = "http://www.p2peye.com/member.php";


    //********************************需要登录的*************************************************
    try {
 Connection.Response res = 
     Jsoup.connect(logURL)
.data("mod","logging"
    ,"action","login"
    ,"loginsubmit","yes"
    ,"loginhash","Lsc66"
    ,"username","puqiuxiaomao"
    ,"password","a1234567")
.method(Method.POST)
.execute();
 
 
 //这儿的SESSIONID需要根据要登录的目标网站设置的session cookie名字而定
 Connection con=Jsoup.connect(listURL);
 //设置访问形式（电脑访问，手机访问）：直接百度都参数设置
 con.header("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)");
 //把登录信息的cookies保存如map对象里面
 Map  map=res.cookies();
 Iterator> it =map.entrySet().iterator();
 while(it.hasNext()){
   Entry en= it.next(); 
   //把登录的信息放入请求里面
   con =con.cookie(en.getKey(), en.getValue());
   
 }
 //再次获取document对象。
 document objectDoc = con.get();
 
 Elements elements = objectDoc.getAllElements();//获取这个连接返回页面的源码内容（不是源码跟源码差不多）
 for (Element element : elements) {
   //element是迭代出来的标签：如：
   Elements elements2= element.getAllElements();//
    for (Element element2 : elements2) {
      element2.text();
      element2.attr("href");//获取标签属性。element2代表a标签：href代表属性
      element2.text();//获取标签文本
   }
 }
 
 //********************************不需要登录的*************************************************
 
 String URL = "http://www.p2peye.com/blacklist.php?p=2";
 document conTemp = Jsoup.connect(URL).get();
 Elements elementsTemps = conTemp.getAllElements();
  for (Element elementsTemp : elementsTemps) {
    elementsTemp.text();
    elementsTemp.attr("href");//获取标签属性。element2代表a标签：href代表属性
    elementsTemp.text();//获取标签文本
 }
 
 
 //********************************ajax方法获取内容。。。*************************************************。
  HttpURLConnection connection = null;
   BufferedReader reader = null;
   try {
     StringBuffer sb = new StringBuffer();
     URL getUrl = new URL(URL);
     connection = (HttpURLConnection)getUrl.openConnection();
     reader = new BufferedReader(new InputStreamReader(
  connection.getInputStream(),"utf-8"));
     String lines;
     while ((lines = reader.readLine()) != null) {
sb.append(lines);
     };
     List> list = parseJSON2List(sb.toString());//json转换成list
   } catch (Exception e) {
     
   } finally{
     if(reader!=null)
try {
  reader.close();
} catch (IOException e) {
}
     // 断开连接
     connection.disconnect();
   }
 
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    
  }
  

  public static Map parseJSON2Map(String jsonStr){ 
    Map map = new HashMap(); 
    //最外层解析 
    JSonObject json = JSONObject.fromObject(jsonStr); 
    for(Object k : json.keySet()){ 
      Object v = json.get(k);  
      //如果内层还是数组的话，继续解析 
      if(v instanceof JSONArray){ 
 List> list = new ArrayList>(); 
 Iterator it = ((JSONArray)v).iterator(); 
 while(it.hasNext()){ 
   JSonObject json2 = it.next(); 
   list.add(parseJSON2Map(json2.toString())); 
 } 
 map.put(k.toString(), list); 
      } else { 
 map.put(k.toString(), v); 
      } 
    } 
    return map; 
  } 
  
  public static List> parseJSON2List(String jsonStr){ 
    JSonArray jsonArr = JSONArray.fromObject(jsonStr); 
    List> list = new ArrayList>(); 
    Iterator it = jsonArr.iterator(); 
    while(it.hasNext()){ 
      JSonObject json2 = it.next(); 
      list.add(parseJSON2Map(json2.toString())); 
    } 
    return list; 
  } 
  
  

}

二，这个是获取验证码的类，可以研究下。（但你要要分析出网站的验证码的请求地址）

package com.minxinloan.black.web.utils;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;

public class Utils {//解析验证码的
  public static Content getRandom(String method, String sUrl,// 要解析的url
      Map paramMap, // 存放用户名和密码的map
      Map requestHeaderMap,// 存放cookie的map
      boolean isOnlyReturnHeader, String path) {

    Content content = null;
    HttpURLConnection httpUrlConnection = null;
    InputStream in = null;
    try {
      URL url = new URL(sUrl);
      boolean isPost = "POST".equals(method);
      if (method == null
   || (!"GET".equalsIgnoreCase(method) && !"POST"
.equalsIgnoreCase(method))) {
 method = "POST";
      }
      URL resolvedURL = url;
      URLConnection urlConnection = resolvedURL.openConnection();
      httpUrlConnection = (HttpURLConnection) urlConnection;
      httpUrlConnection.setRequestMethod(method);
      httpUrlConnection.setRequestProperty("Accept-Language",
   "zh-cn,zh;q=0.5");
      // Do not follow redirects, We will handle redirects ourself
      httpUrlConnection.setInstanceFollowRedirects(false);
      httpUrlConnection.setDoOutput(true);
      httpUrlConnection.setDoInput(true);
      httpUrlConnection.setConnectTimeout(5000);
      httpUrlConnection.setReadTimeout(5000);
      httpUrlConnection.setUseCaches(false);
      httpUrlConnection.setDefaultUseCaches(false);
      httpUrlConnection.connect();

      int responseCode = httpUrlConnection.getResponseCode();

      if (responseCode == HttpURLConnection.HTTP_OK
   || responseCode == HttpURLConnection.HTTP_CREATED) {
 byte[] bytes = new byte[0];
 if (!isOnlyReturnHeader) {
   DataInputStream ins = new DataInputStream(
httpUrlConnection.getInputStream());
   // 验证码的位置
   DataOutputStream out = new DataOutputStream(
new FileOutputStream(path + "/code.bmp"));
   byte[] buffer = new byte[4096];
   int count = 0;
   while ((count = ins.read(buffer)) > 0) {
     out.write(buffer, 0, count);
   }
   out.close();
   ins.close();
 }
 String encoding = null;
 if (encoding == null) {
   encoding = getEncodingFromContentType(httpUrlConnection
.getHeaderField(""));
 }
 content = new Content(sUrl, new String(bytes, encoding),
     httpUrlConnection.getHeaderFields());
      }
    } catch (Exception e) {
      return null;
    } finally {
      if (httpUrlConnection != null) {
 httpUrlConnection.disconnect();
      }
    }
    return content;
  }

  public static String getEncodingFromContentType(String contentType) {
    String encoding = null;
    if (contentType == null) {
      return null;
    }
    StringTokenizer tok = new StringTokenizer(contentType, ";");
    if (tok.hasMoreTokens()) {
      tok.nextToken();
      while (tok.hasMoreTokens()) {
 String assignment = tok.nextToken().trim();
 int eqIdx = assignment.indexOf('=');
 if (eqIdx != -1) {
   String varName = assignment.substring(0, eqIdx).trim();
   if ("charset".equalsIgnoreCase(varName)) {
     String varValue = assignment.substring(eqIdx + 1)
  .trim();
     if (varValue.startsWith(""")
  && varValue.endsWith(""")) {
// substring works on indices
varValue = varValue.substring(1,
    varValue.length() - 1);
     }
     if (Charset.isSupported(varValue)) {
encoding = varValue;
     }
   }
 }
      }
    }
    if (encoding == null) {
      return "UTF-8";
    }
    return encoding;
  }

  // 这个是输出
  public static boolean inFile(String content, String path) {
    PrintWriter out = null;
    File file = new File(path);
    try {
      if (!file.exists()) {
 file.createNewFile();
      }
      out = new PrintWriter(new FileWriter(file));

      out.write(content);
      out.flush();
      return true;
    } catch (Exception e) {
      e.printStackTrace();
    } finally {
      out.close();
    }
    return false;
  }

  public static String getHtmlReadLine(String httpurl) {
    String CurrentLine = "";
    String TotalString = "";
    InputStream urlStream;
    String content = "";

    try {
      URL url = new URL(httpurl);

      HttpURLConnection connection = (HttpURLConnection) url
   .openConnection();

      connection.connect();
      System.out.println(connection.getResponseCode());
      urlStream = connection.getInputStream();

      BufferedReader reader = new BufferedReader(

      new InputStreamReader(urlStream, "utf-8"));

      while ((CurrentLine = reader.readLine()) != null) {
 TotalString += CurrentLine + "n";
      }

      content = TotalString;

    } catch (Exception e) {
    }

    return content;
  }
}


class Content {
  private String url;
  private String body;
  private Map> m_mHeaders = new HashMap>();

  public Content(String url, String body, Map> headers) {
    this.url = url;
    this.body = body;
    this.m_mHeaders = headers;
  }

  public String getUrl() {
    return url;
  }

  public String getBody() {
    return body;
  }

  public Map> getHeaders() {
    return m_mHeaders;
  }

}