代码如下:
import asyncio
import pyppeteer as pyp
async def antiAntiCrawler(page):
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36')
await page.evaluateOnNewDocument(
'()=>{Object.defineProperties(navigotor,{webdriver:{get:()=>false}})}'
)#反爬虫设置
async def getOjSourceCode(loginUrl):
width,height=1200,800
browser=await pyp.launch(headless=False, #browser伪装成浏览器并且自动打开
userdataDir="E:/python代码/图片爬虫",
args=[f'--window-size={width},{height}'])
page=await browser.newPage()#浏览器生成后创建一个新的页面
await antiAntiCrawler(page)
await page.setViewport({'width':width,'height':height})
await page.goto(loginUrl) # 浏览器在登录页面,输入登录
# #若手动登录,则以下若干行可以去掉
# element=await page.querySelector("#root > div > main > div > div > div > div > div.SignContainer-content > div > div:nth-child(1) > form > div.SignFlow-account > div > label > input")
# await element.type("xxxx")
# element=await page.querySelector("#root > div > main > div > div > div > div > div.SignContainer-content > div > div:nth-child(1) > form > div.SignFlow-password > div > div")
# await element.type("xxxx")
# element=await page.querySelector("#root > div > main > div > div > div > div > div.SignContainer-content > div > div:nth-child(1) > form > button")
# await element.click()
#做等待,等待click的操作时间
await page.waitForSelector("#root > div > div:nth-child(2) > header > div.AppHeader-inner.css-qqgmyv > ul > li:nth-child(5) > a",timeout=40000)
#等待"回答问题"出现,timeout=30000,毫秒,如果等待时间太久没有出现回答问题的界面,程序不能进行下去设置的时间限制
element=await page.querySelector("#Popover15-toggle")#找个人头像
await element.click()#点击
await page.waitForSelector("#Popover15-content > div > a:nth-child(1)")#等待”我的主页“
element = await page.querySelector("#Popover15-content > div > a:nth-child(1)")
await element.click()#点击
await page.waitForFunction()#等待网页装入完毕
element=await page.querySelectorAll(".ContentItem-title")
page2=await browser.newPage()#新打开一个页面
await antiAntiCrawler(page2)
for element in element[:10]:
obj=await element.getProperty("href")
url=await obj.jsonValue()
await page2.goto(url)
element=await element.getProperty("span")#查找span tag
obj=await element.getProperty("innerText")
text=await obj.jsonValue()
print(text)
print("-----------------")
await browser.close()
def main():
url="https://www.zhihu.com/"
asyncio.get_event_loop().run_until_complete(getOjSourceCode(url))
main()
出现的问题:
需要点击的内容:
点击后出现:
但是点击错误,不知道为啥,有没有大佬能帮帮忙啊



