栏目分类:
子分类:
返回
名师互学网用户登录
快速导航关闭
当前搜索
当前分类
子分类
实用工具
热门搜索
名师互学网 > IT > 软件开发 > 后端开发 > Java

【精品】爬取 国家统计局 2020年 省市县乡村 数据

Java 更新时间: 发布时间: IT归档 最新发布 模块sitemap 名妆网 法律咨询 聚返吧 英语巴士网 伯小乐 网商动力

【精品】爬取 国家统计局 2020年 省市县乡村 数据

import javax.annotation.Resource;

import java.io.IOException;

import java.util.ArrayList;

import java.util.List;

@Slf4j

@SpringBootTest

public class InitAdd5Tables {

private Document connect(String url) {

if (url == null || url.isEmpty()) {

throw new IllegalArgumentException(“无效的url”);

}

try {

return Jsoup.connect(url).timeout(100 * 1000).get();

} catch (IOException e) {

System.out.println(url+“地址不存在”);

return null;

}

}

public List getProvinces() {

List res = new ArrayList<>();

Document connect = connect(“http://localhost:8080/2020/default.htm”);

Elements rowProvince = connect.select(“tr.provincetr”);

for (Element provinceElement : rowProvince) {// 遍历每一行的省份城市

Elements select = provinceElement.select(“a”);

for (Element province : select) {// 每一个省份(四川省)

String name = province.text();

String code = province.select(“a”).attr(“href”);

res.add(code.substring(0, code.lastIndexOf(“.”)) + “*” + name);

}

}

return res;

}

@Test

public void testGetProvince() {

getProvinces().forEach(System.out::println);

}

@Resource

private ProvinceMapper provinceMapper;

@Test

void insertProvinces() {

List list = new ArrayList<>();

for (String p : getProvinces()) {

String[] split = p.split(“*”);

Province province = Province.builder().code(split[0]).name(split[1]).build();

list.add(province);

}

//list.forEach(System.out::println);

int res = provinceMapper.batchInsert(list);

System.out.println(res);

}

public List getCitiesByProvince(String provinceCode) {

List res = new ArrayList<>();

Document connect = connect(“http://localhost:8080/2020/” + provinceCode + “.html”);

Elements rowCity = connect.select(“tr.citytr”);

for (Element cityElement : rowCity) {// 遍历每一行的省份城市

String name = cityElement.select(“td”).text();

String[] split = name.split(" ");

res.add(split[0].substring(0, 4) + “*” + split[1]);

}

return res;

}

@Test

public void testGetCitiesByProvince() {

getCitiesByProvince(“41”).forEach(System.out::println);

}

@Resource

private CityMapper cityMapper;

@Test

void insertCities() {

List pList = getProvinces();

for (String p : pList) {

List list = new ArrayList<>();

String[] split = p.split(“*”);

List cList = getCitiesByProvince(split[0]);

Province pp = provinceMapper.selectOne(new QueryWrapper().eq(“code”, split[0]));

for (String c : cList) {

String[] tmp = c.split(“*”);

City city = City.builder().name(tmp[1]).code(tmp[0]).provinceId(pp.getId()).build();

//System.out.println(city);

list.add(city);

}

//一个省一个省的添加

int res = cityMapper.batchInsert(list);

System.out.println(res);

}

}

public List getCountriesByCity(String cityCode) {

List res = new ArrayList<>();

Document connect = connect(“http://localhost:8080/2020/” + cityCode + “.html”);

Elements rowCountry = connect.select(“tr.countytr”);

if (rowCountry.size() == 0) {

Elements townCountry = connect.select(“tr.towntr”);

for (Element townElement : townCountry) {

String txt = townElement.select(“td”).text();

String[] split = txt.split(" ");

res.add(split[0].substring(0, 9) + “*” + split[1]);

//比如海南省下的儋州市,只有4级目录,没有country

}

} else {

for (Element countryElement : rowCountry) {// 遍历每一行的省份城市

String txt = countryElement.select(“td”).text();

String[] split = txt.split(" ");

res.add(split[0].substring(0, 6) + “*” + split[1]);

}

}

return res;

}

@Test

void testGetCountiesByProvince() {

getCountriesByCity(“46/4604”).forEach(System.out::println);

}

@Resource

private CountryMapper countryMapper;

@Test

void inser 《大厂前端面试题解析+Web核心总结学习笔记+企业项目实战源码+最新高清讲解视频》无偿开源 徽信搜索公众号【编程进阶路】 tCountry() {

List pList = getProvinces();

for (int i = 0; i < pList.size(); i++) {

String p = pList.get(i);

String[] split = p.split(“*”);

//System.out.println(split[0] +" "+split[1]); // 13 河北省

List cList = getCitiesByProvince(split[0]);

for (String c : cList) {

String[] split2 = c.split(“*”);

//System.out.println(split2[0] +" * "+split2[1]);//1301 * 石家庄市

List c2List = getCountriesByCity(split2[0].substring(0, 2) + “/” + split2[0]);

City city = cityMapper.selectOne(new QueryWrapper().eq(“code”, split2[0]));

List list = new ArrayList<>();

for (String c2 : c2List) {

String[] split3 = c2.split(“*”);

System.out.println(split3[0] + " * " + split3[1]);

Country country = Country.builder().name(split3[1]).code(split3[0]).cityId(city.getId()).build();

list.add(country);

}

int res = countryMapper.batchInsert(list);

System.out.println(res);

}

}

}

public List getTownsByCountry(String countryCode) {

List res = new ArrayList<>();

Document connect = connect(“http://localhost:8080/2020/” + countryCode + “.html”);

if (connect != null) {

Elements rowTown = connect.select(“tr.towntr”);

if (rowTown.size() == 0) {

Elements rowVillage = connect.select(“tr.villagetr”);

for (Element villageElement : rowVillage) {

String txt = villageElement.select(“td”).text();

String[] split = txt.split(" ");

res.add(split[1] + “*” + split[2]);

//比如海南省下的儋州市,只有4级目录,没有country

}

} else {

for (Element townElement : rowTown) {// 遍历每一行的省份城市

String txt = townElement.select(“td”).text();

String[] split = txt.split(" ");

res.add(split[0].substring(0, 9) + “*” + split[1]);

}

}

}

return res;

}

转载请注明:文章转载自 www.mshxw.com
本文地址:https://www.mshxw.com/it/930806.html
我们一直用心在做
关于我们 文章归档 网站地图 联系我们

版权所有 (c)2021-2022 MSHXW.COM

ICP备案号:晋ICP备2021003244-6号