利用Python中的Xpath获取节点爬虫

以FreeBuf的首页为例爬取链接和标题

import requests
from lxml import etree

headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:94.0) Gecko/20100101 Firefox/94.0'
    }

def getTree(url):
    res = requests.get(url,headers,timeout = 5)
    # 自动分析网页内容的编码方式
    res.encoding = res.apparent_encoding
    res_text = res.text
    # 解析网页
    tree = etree.HTML(res_text)
    return tree


url = "https://www.freebuf.com/"
xml = getTree(url)

#获取节点href内容，//表示任意位置定位，/表示从根结点定位，@后为属性名
url_links = xml.xpath('//div[@]//div[@]//div[@]//a[1]/@href')

#text()方法获取节点span所有内容
span = xml.xpath('//div[@]//div[@]//div[@]//a[1]/span/text()')
print(span)
print(url_links)
url_link = []

for i in url_links:
    link = url+i
    url_link.append(link)
print(url_link)