ubuntu/+cdmirrors 爬虫

import requests
from lxml import etree
#//lxml需要下载(pip install lxml)

def getone2( url):
    try:

        text = requests.get(url, timeout=2).text
        html = etree.HTML(text)
        #获取到网页中的html
        hrefs = html.xpath("//a/@href")
        f=open('list2.txt', 'a+')
        #f.write(url +'=================================n')
        for href in hrefs:
            if (str(href).endswith("iso")  ) :
                f.write(url + href+ 'n')
        f.close
    except Exception as e:
        print(str(e))


def getone( url):
    try:

        text = requests.get(url, timeout=2).text
        html = etree.HTML(text)
        #获取到网页中的html
        hrefs = html.xpath("//a/@href")
        f=open('list1.txt', 'a+')
        #f.write(url +'=================================n')
        for href in hrefs:
            if (str(href).find("20.04.3") > -1 ) :
                url1= url + href
                getone2(url1)
                f.write(url + href+ 'n')
        f.close
    except Exception as e:
        print(str(e))

url = "https://launchpad.net/ubuntu/+cdmirrors"
#header={"user-ange":"xxxxxx"}
#text = requests.get(url, headers=header).text
text = requests.get(url).text
html = etree.HTML(text)
#获取到网页中的html
hrefs = html.xpath("//a/@href")
f=open('list.txt', 'w')
#f=open('list.txt', 'a+')
#f.write('n'.join(hrefs))
for href in hrefs:
    print(hrefs.index(href) + 1, len(hrefs), href)
    print(str(href).find("launchpad.net/"))
    if (str(href).find("launchpad.net/") < 0 ) and (str(href).startswith('http')):
        f.write(href+'n')
        getone(href)

f.close