python 爬虫学习日记（3）

2022.7.25

当作日后复习爬虫的笔记

# -*- coding: utf-8 -*-
"""
Created on Fri Jul 22 11:02:15 2022

@author: rosie are rosie
"""
import urllib.request
import urllib.parse#解析器，将中文、英语按照一定格式解析

#获得一个get请求
# response=urllib.request.urlopen("https://bbs.hupu.com/pubg")
# print(response.read().decode('utf-8'))
#decode解码，对解析到网页的数据用utf-8（防止中文乱码）的方式解析


# #获得一个post 请求;post:模拟用户真实登陆

# data=bytes(urllib.parse.urlencode({"hello":"world"}),encoding='utf-8')
# #bytes转成二进制数据包
# #urlencode封装
# #encoding='utf-8'：按照'utf-8'封装
# response=urllib.request.urlopen("http://httpbin.org/post",data=data)
# #data传给form
# print(response.read().decode('utf-8'))
'''
输出结果：
{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "hello": "world"
  }, #由data传送
  "headers": {
    "Accept-Encoding": "identity", 
    "Content-Length": "11", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "httpbin.org", 
    "User-Agent": "Python-urllib/3.7", 
    "X-Amzn-Trace-Id": "Root=1-62dd2ed1-6b0e6cea53e452de7b88877e"
  }, 
  "json": null, 
  "origin": "58.217.108.232", 
  "url": "http://httpbin.org/post"
}
'''

#延时处理
# try:
#     response=urllib.request.urlopen("http://httpbin.org/get",timeout=0.01)
#     print(response.read().decode('utf-8'))
# except Exception as result:
#     print("timeout error")

'''
{
  "args": {}, 
  "headers": {
    "Accept-Encoding": "identity", 
    "Host": "httpbin.org", 
    "User-Agent": "Python-urllib/3.7" # 没有伪装成浏览器，直接被识别出来是爬虫
    "X-Amzn-Trace-Id": "Root=1-62dd3b97-4180b93915bab4d930b731db"
  }, 
  "origin": "58.217.108.232", 
  "url": "http://httpbin.org/get"
}
'''



# response=urllib.request.urlopen("https://baidu.com")
# # print(response.status)
# #输出状态码 200:请求已成功 418:被网站的反爬程序返回的
# # print(response.getheaders())
# print(response.getheader("Bdpagetype"))#想看网页中某一个属性header不用加s
'''
result:
Traceback (most recent call last):
当用快速爬取某网站出现经常出现Traceback (most recent call last):的错误，
也就是连接失败

raise HTTPError(req.full_url, code, msg, hdrs, fp):
    网站存在反爬虫机制，爬虫未进行UA封装，拒绝了爬虫请求
'''


#模拟浏览器：
url="http://httpbin.org/post"
# url="https://baidu.com/post"
data=bytes(urllib.parse.urlencode({"name":"spider"}),encoding="utf-8")#解析
headers={
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.134 Safari/537.36 Edg/103.0.1264.71"      
}

req=urllib.request.Request(url=url,data=data,headers=headers,method="post")#请求对象
response=urllib.request.urlopen(req)#响应对象
print(response.read().decode('utf-8'))

最后模拟浏览器没跑出来，留个坑，报错：
HTTPError: Bad Request

douban ip好像被ban了