python基础爬虫

import requests

def html(web):
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'}
    url = web
    response = requests.get(url = url,headers = headers)
    txt = response.text
    with open('data.txt','w',encoding = 'utf-8') as fi:
        fi.write(txt)
# 获取网页源代码

def gettxt(txt_1):
    fi = open(txt_1, 'r', encoding = 'utf-8')
    lines = fi.readlines()
    fi.close()
    return lines
# 读取保存在本地的txt文档

def src(lines):
    dx_logos = []
    dxs = []
    for i in range(len(lines)):
        if r'src="https://' and '.jpg' in lines[i]:  # 判断是否为jpg格式
            if 'alt="'in lines[i+1]:  # png图片的下一行是否为学校名称
                dx_logos.append(lines[i].split('"')[1])  # 数据切割，留下需要的字段
                dxs.append(lines[i+1].split('alt="')[1].split('"')[0])  # 同上
        elif r'src="https://' and '.png' in lines[i]:  # 判断是否为jpg格式
            if 'alt="'in lines[i+1]:  # png图片的下一行是否为学校名称
                dx_logos.append(lines[i].split('"')[1])  # 数据切割，留下需要的字段
                dxs.append(lines[i+1].split('alt="')[1].split('"')[0])  # 同上
    return dx_logos, dxs
# 获取大学logo图片ulr和大学名称

def clean(dx_logos, dxs):
    fo = open('data_1.txt', 'w')
    for i in range(len(dx_logos)):
        if '?' in dx_logos[i]:  # 部分图片后缀有不需要的数据，判断‘？’是否在内，若在，以‘？’作切割提取有用的数据
            dx_logo = dx_logos[i].split('?')[0]  # 以‘？’进行切割
            fo.write(dxs[i] + 'n')  # 写入下一行的大学名
            fo.write(dx_logo + 'n')  # 大学名第二行紧跟大学logo图片
        else:
            fo.write(dxs[i] + 'n')
            fo.write(dx_logos[i] + 'n')  # 没有问号的直接写入
    fo.close()
# 数据清洗并保存在data_1的文件夹中

def photo(txt_2):
    fi_1 = open(txt_2, 'r')
    datas = fi_1.readlines()
    for i in range(len(datas)):
        if 'http' in datas[i]:
            url = datas[i].strip()
            response = requests.get(url=url)
            content = response.content
            fi = open(r'E:python_studylogo{}.jpg'.format(datas[i-1].strip()), 'wb')
            fi.write(content)
            fi.close()
# 将获取的代码生成图片

def main():
    txt_1 =  html('https://www.icourse163.org/university/view/all.htm#/')
    lines = gettxt('data.txt')
    dx_logos, dxs = src(lines)
    txt_2 = clean(dx_logos, dxs)
    photofile = photo('data_1.txt')
main()
# 使用main()方法运行函数