批量爬取网站图片

相关模块或技术:
爬虫模块:requests
数据解析:BeautifulSoup 和 正则re

初步请求获取响应

import requests
from bs4 import BeautifulSoup
import re

def get_page(url):

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36'
    }
    try:
        response = requests.get(url,headers=headers)
    except:
        print('请求失败！')
    else:
        return response

if __name__ == '__main__':
    url = ''
    response = get_page(url)
    print(response)

完整代码

import requests
from bs4 import BeautifulSoup
import re

def get_page(url):

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36'
    }
    try:
        response = requests.get(url,headers=headers)
    except:
        print('请求失败！')
    else:
        return response

def get_link_name(url):
    name_list = []
    link_list = []
    regular = ''
    link_header = ''
    response = get_page(url)
    html_doc = response.text
    soup = BeautifulSoup(html_doc,'lxml')
    content = soup.select('.image-package')
    print(content)
    for item in content:
        name = item.find('div',{'class':'image-caption'}).string
        name_list.append(name)
        text = str(item.select('.image-view'))
        result = re.match(regular,text)
        link = result.group(1)
        link_list.append(link_header+link)
    return link_list,name_list

def load_img(link_list,name_list):
    for link,name in zip(link_list,name_list):
        response = get_page(link)
        with open(f'./images/{name}','wb') as file:
            file.write(response.content)
        print(f'{name}图片下载成功!')

def main():
    url = ''
    link_list,name_list = get_link_name(url)
    load_img(link_list,name_list)

if __name__ == '__main__':
    main()