csdn 涨粉攻略 代码(三)关注数 webmagic 爬虫

获取关注者数目 

package com.web;

import java.util.List;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.JsonPathSelector;


public class FollowCountSpider implements PageProcessor {

    

    private final String TAG = FollowCountSpider.class.getSimpleName();


    Integer count = 0;


    private Site site = Site
            .me()
            .setDomain("blog.csdn.net")
            .setSleepTime(1000)
            // 便于测试，休眠较长时间。
            .setUserAgent(
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");

    @Override
    public void process(Page page) {
        String rawText = page.getRawText();
        String data = new JsonPathSelector("$.data").select(rawText);
        String replace = data.replace("{list=", "{"list":");
        List selectList = new JsonPathSelector("$.list").selectList(replace);
        count = selectList.size();
    }

    @Override
    public Site getSite() {
        return site;
    }

}

测试类

package com.web;

import us.codecraft.webmagic.Spider;


public class T2 {
    public static void main(String[] args) {
        String username = "";
        String pageIndex = "1";
        int followCount = 0;
        for (int i = 1; ; i++) {
            String fanUrl =
                    "https://blog.csdn.net/community/home-api/v1/get-follow-list?page=" + i + "&size=20&noMore=false&blogUsername=" + username;
            FollowCountSpider followCountSpider = new FollowCountSpider();
            Spider.create(followCountSpider).addUrl(fanUrl).thread(1).run();
            if (followCountSpider.count < 20) {
                followCount = (i-1) * 20 + followCountSpider.count;
                break;
            }
        }
        System.out.println(followCount);

    }
}