import requests # 导入requests库爬取网页源代码
def html(web):
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'} ## 此处'User-Agent'味谷歌浏览器的'User-Agent',每个浏览器不同
url = web
response = requests.get(url = url,headers = headers)
txt = response.text
with open('taobao.txt','w',encoding = 'utf-8') as fi:
fi.write(txt)
# 爬取网页源代码
def gettxt(txt):
f = open(txt,'r',encoding = 'utf-8')
lines = f.readlines()
f.close()
return lines
# 读取保存在本地的txt文件
def href(lines):
urls = []
for line in lines:
if ' in line:
url = line.split('href="')[1].split('"')[0]
if 'https://' or 'http://'in url:
urls.append(url)
return urls
# 解析并提取其中的图片链接
def showRe(urls):
count = 0
for url in urls:
print('第{:2}个URL:{}'.format(count,url))
count += 1
# 输出提取结果到屏幕
def saveRe(file,urls):
with open(file, 'w') as f:
for url in urls:
f.write(url + 'n')
f.close()
# 保存提取结果文件
def main():
txt = html('https://www.taobao.com/')
lines = gettxt('taobao.txt')
urls = href(lines)
showRe(urls)
saveRe('data.txt',urls)
main()
# 部分功能代码失陷后,需要编写main()函数,调用各个功能函数
运行结果如下
第 0个URL:https://www.google.cn/intl/zh-CN/chrome/browser/desktop/ 第 1个URL:http://www.uc.cn/ucbrowser/download/ 第 2个URL://www.taobao.com 第 3个URL://www.taobao.com 第 4个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=连衣裙&refpid=420460_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f66 第 5个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=四件套&refpid=420461_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f67 第 6个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=T恤&refpid=420462_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f67 第 7个URL:https://s.taobao.com/search?q=%E5%A5%B3%E9%9E%8B&refpid=420466_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f68 第 8个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=短裤&refpid=430145_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f69 第 9个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=半身裙&refpid=430146_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f70 第10个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=男士外套&refpid=430147_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f70 第11个URL:https://s.taobao.com/search?q=墙纸&refpid=430148_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f70 第12个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=行车记录仪&refpid=420467_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f69 第13个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=男鞋&refpid=430144_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f69 第14个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=耳机&refpid=420463_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f67 第15个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=女包&refpid=420464_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f68 第16个URL:http://s.taobao.com/search?spm=1.7274553.1997520241-2.2.TpEKPQ&q=沙发&refpid=420465_1006&source=tbsy&style=grid&tab=all&pvid=d0f2ec2810bcec0d5a16d5283ce59f68 第17个URL:https://market.m.taobao.com/app/fdilab/download-page/main/index.html



