python--requests模块

import requests
head = { #存放需要伪装的头信息
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
}
pram = {
    "type": "13",
    "interval_id": "100:90",
    "action": "",
    "start": "0",
    "limit": "20",
}
url = 'https://xxxxxxxxxxxxxx'
response = requests.get(url=url,headers=head,params=pram)
#获取响应数据
#json()可以将获取到的json格式的字符串进行反序列化
page_text = response.json()
fp = open('./douban.txt','w')
for dic in page_text:
    title = dic['title']
    score = dic['score']
    fp.write(title+':'+score+'n')

import requests
head = { #存放需要伪装的头信息
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
}
#post请求的请求参数
data = {
    "cname": "",
    "pid": "",
    "keyword": "天津",
    "pageIndex": "1",
    "pageSize": "10",
}
#在抓包工具中：Form Data存放的是post请求的请求参数，而Query String中存放的是get请求的请求参数
url = 'http://www.xxxxxxxxxxxxx.keyword'
#在post请求中，处理请求参数的是data这个参数不是params
response = requests.post(url=url,headers=head,data=data)
response = requests.post(url=url,headers=head,json=data) # 需要json格式时就使用json
page_text = response.json()
for dic in page_text['Table1']:
    name = dic['storeName']
    addr = dic['addressDetail']
    print(name,addr)

# requests编写稍微麻烦，但是可以实现UA伪装
import requests
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36X-Requested-With: XMLHttpRequest'
}
#获取图片地址：通常为img标签的src属性值
img_src = 'http://img.itlun.cn/uploads/allimg/180506/1-1P5061TS6-lp.jpg'
#对图片发起请求
response = requests.get(url=img_src,headers=headers)
#获取图片数据:content返回的是二进制形式的响应数据
img_data = response.content
#持久化存储
with open('./123.jpg','wb') as fp:
    fp.write(img_data)

======================================

# urllib 编写简单，但是无法实现UA伪装
from urllib import request
img_src = 'http://img.itlun.cn/uploads/allimg/180506/1-1P5061TS6-lp.jpg'
#urlretrieve可以直接对图片发起请求，且将请求到的图片数据进行持久化存储
request.urlretrieve(img_src,'./123.jpg')

import requests
main_url = 'http://md.itlun.cn/a/nhtp/'
response = requests.get(url=main_url)
#处理页面中的中文乱码
response.encoding = 'gbk' # 一般就是gbk，或者utf-8
#获取了页面源码数据
page_text = response.text