python爬取猫眼电影

爬取猫眼电影排名前一百的电影
验证问题不知道怎么解决，隔段时间要重新输一次链接‘

# -*- coding: utf-8 -*-
"""
Created on Wed Dec 29 21:07:41 2021

@author: shiyu
"""

import requests
import re

#爬取电影名
def get(url):
    try:
        html=requests.get(url,headers=headers)
    except:
        print('error')
    if html.text.find('验证')!=-1:
        print('需要验证，重新输入网址')
    k2=r'(

    ans=re.findall(k2, html.text)
    if ans:
        for it in ans:
            list.append(it[1])

url='https://www.maoyan.com/board/4?timeStamp=1640784403769&sVersion=1&offset=0&requestCode=8202caf044f2e49747af494993ef4b1fe5shf&index=3&webdriver=false&signKey=1a9bca8888a3481620062a6d57df6544&channelId=40011'
headers={'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.62'
         }
list=[]


#将可能出现在两种位置的offset=**替换掉,得到‘中间url’
k1_1=r'(.*?)(offset=.*?)(&.*)'
l=re.search(k1_1,url)
if l:
    s=re.sub(k1_1,r'1%s' %'{}'+l.group(3),url)
else:
    k1_2=r'offset=.*'
    s=re.sub(k1_2,'{}',url)
url_=s

#清空data.txt
with open('data.txt','w') as f:
    pass
#生成偏移值得到下一个网页链接并爬取
for i in range(0,10):
    add='offset='+str(i*10)
    url=url_.format(add)
    print(url+'n')
    get(url)

#爬取结果写入data.txt
with open('data.txt','a',encoding='utf-8') as f:
    for it in list:
        f.write(it+'n')