python requests 使用整理

• • 设置代理
  • requests 使用bs4

下面展示一些 内联代码片。

def get_proxy():
	# 获取代理ip和port
    proxy_url = "http://api.wandoudl.com/api/ip?app_key=app_key&pack=num&num=1&xy=1&type=2&lb=n&nr=1&area_id=undefined"
    proxy = None
    response = requests.get(proxy_url)
    if response.status_code == 200:
        res = response.text
        res_json = json.loads(res)
        proxy = str(res_json["data"][0]["ip"])+":"+str(res_json["data"][0]["port"])
    return proxy 

# 设置代理
http_proxy_server = get_proxy()
proxies = {
    "https": 'http://{}' . format(http_proxy_server),
}
target_url = "https://"
resp = requests.get(target_url ,proxies=proxies)

from bs4 import BeautifulSoup
import requests
response = requests.get(url)
soup = bs4.BeautifulSoup(response.text,'lxml')
title = soup.title.text
for line in soup.find_all("p"):
    data_line =  line.text