1.正则匹配时犯的错误
2.爬虫时遇到的反爬机制
3.文件名因为粗心导致debug了两个小时

import requests
import re
import os
import random
import time
import threading
import urllib3
from urllib.parse import quote

urllib3.disable_warnings()
headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"
}  # 常规请求头
up_url = "https://new.abb.com/products/robotics/zh/industrial-robots/"
up_url1 = "https://new.abb.com"

# 定义一个生成四位随机数并×当前的秒数的函数
def get_random_num():
    return str(random.randint(1, 9) * int(time.time()))
# 文件夹创建
def if_exists(path):
    if not os.path.exists(path):
        os.makedirs(path)
# 获取所有的类型
def get_mian_rb():
    mian_url = "https://new.abb.com/products/robotics/zh/industrial-robots"  # 首页地址
    robot_type_all = requests.get(mian_url, headers=headers, verify=False)
    re_rb = re.findall('http://kaotopstatic.cdn.bcebos.com/skin/sinaskin/image/nopic.gif',
                       robot_type_all.text)  # 获取所有的机器人类型
    re_url = re.findall(', robot_type_all.text)  # 获取所有的机器人类型的链接
    del re_rb[0]  # 删除第一个元素
    del re_url[0]  # 删除第一个元素
    print("全部机器人已获取成功")
    return re_rb, re_url  # 返回所有的机器人类型和链接
# 获取所有的图片的连接
def get_png_url(rb_url):
    response = requests.get(up_url1 + rb_url, headers=headers, verify=False)
    rb_photo = re.findall(
        " ",
        response.text)  # 图片库
    rb_one_photo = re.findall('
,
                              response.text)  # 首页横屏图片
    pid = re.findall('cid":"http://kaotopstatic.cdn.bcebos.com/skin/sinaskin/image/nopic.gif"', response.text)  # 访问js的pid
    if len(rb_photo) != 0:
        print("已获取" + rb + "的图片库" + rb_photo[0])
    if len(rb_one_photo) != 0:
        print("已获取" + rb + "的首页横屏图片" + rb_one_photo[0])
    else:  # 如果方法1没成功调用方法2再次匹配
        rb_one_photo = re.findall(',
                                  response.text)  # 首页横屏图片
    if len(pid) != 0:
        print("已获取" + rb + "的pid" + pid[0])
    return rb_photo, rb_one_photo, pid
# 获取pdf的连接
def get_pdf_url(js_url, url):
    url = quote(url, 'utf-8')  # 防止机器人名称中中文， 进行编码
    js_headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
        "Referer": url  # 防盗链操作
    }
    response = requests.get(js_url, headers=js_headers, verify=False)
    pdf_info = re.findall(
        'RepresentationFileName": "http://kaotopstatic.cdn.bcebos.com/skin/sinaskin/image/nopic.gif"[sS]*?"Title": "http://kaotopstatic.cdn.bcebos.com/skin/sinaskin/image/nopic.gif"[sS]*?"DocumentURL": "http://kaotopstatic.cdn.bcebos.com/skin/sinaskin/image/nopic.gif"[sS]*?RepresentationFileSize',
        response.text)
    print("PDF资料已获取成功")
    pdf_url_list1 = []
    pdf_name_list1 = []
    file_type_list1 = []
    for i in pdf_info:
        # 判断i包含 =zh关键字 也就是中文文档
        if 'zh' in i[2]:
            pdf_url_list1.append(i[2])
            pdf_name_list1.append(i[1])
            file_type_list1.append(i[0])
    return pdf_url_list1, pdf_name_list1, file_type_list1
# 下载图片
def down_png(png_url, rb):
    if png_url is None:  # 如果是空的直接退出
        return
    response = requests.get(png_url, headers=headers, verify=False)
    with_file = "ABB资源\" + rb + "\" + get_random_num() + ".png"  # 文件存放路径
    with open(with_file, "wb") as f:
        f.write(response.content)
        print("已下载成功" + rb + "的图片 路径为：" + with_file)
# 下载pdf
def down_pdf(pdf_url, pdf_name, rb, url):
    url = quote(url, 'utf-8')  # 防止机器人名称中中文， 进行编码
    js_headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
        "Referer": url  # 防盗链操作
    }
    response = requests.get(pdf_url, headers=js_headers, verify=False)
    down_url = re.findall('