开发环境：Windows，Pycharm，Request，BeautifulSoup，urllib
需要一定的 Python 爬虫、HTML 基础
本次要爬取的网站依然是 帅啊 网
我们需要把整站的详情页图片下载到本地
制作爬虫
由于获取下来的 html 编码格式不对，所以要指定编码格式为 utf-8
获取页面中每个 item 的详情页链接
获取详情页的所有图片链接（单个或多个）
以 item 的标题为文件夹，将详情页的图片下载到该文件夹中
抓取下一页（重复 2、3、4 步骤）
from bs4 import BeautifulSoupimport requestsimport osimport urllib.requestimport time
headers = { "cookie": "gsScrollPos-1702684443=0; UM_distinctid=16685e0279d3e0-06f34603dfa898-36664c08-1fa400-16685e0279e133; bdshare_firstime=1539844405694; _d_id=6556c25e9ddd0e7e71096a1e343f6b; gsScrollPos-1702684407=; CNZZDATA1254092508=1744643453-1539842703-%7C1540364765", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
}
path = "D://images/"def get_links(url):
wb_data = requests.get(url, headers=headers) # headers 伪装
wb_data.encoding = "utf-8"
soup = BeautifulSoup(wb_data.text, 'lxml') if wb_data.status_code == 404: return
if not os.path.exists(path): # 判断该文件夹是否存在，不存在则创建
os.mkdir(path)
links = soup.select(".item-img") for link in links:
download_img(link.get('href'))
time.sleep(1) # 下一页
next_page = soup.select(".next")[0].get("href")
print("------ next page -------")
get_links(next_page)
print("------ download done -------")def download_img(url):
wb_data = requests.get(url, headers=headers)
wb_data.encoding = "utf-8"
soup = BeautifulSoup(wb_data.text, 'lxml')
images = soup.select(".wr-single-content-list img")
catalog = soup.select("h1")[0].get_text() # 获取详情页标题作为文件夹名称
catalog = path + catalog + "/"
if not os.path.exists(catalog):
os.mkdir(catalog) for index, image in enumerate(images): # enumerate 是 Python 的内置函数，用它可以同时返回索引和值
print(index)
img = image.get("src")
urllib.request.urlretrieve(img, catalog + str(index) + ".jpg")
print("-------- downloading ---------")if __name__ == "__main__":
get_links("http://www.shuaia.net/index.html")开始爬取
作者：飘渺云轩
链接：https://www.jianshu.com/p/1fed552cd202
