云健康网-热量查询
导入所需的库和模块from gevent import monkey import gevent,requests,bs4,openpyxl,time from gevent.queue import Queue from openpyxl import load_workbook,Workbook,worksheet import re import numpy as np import pandas as pd from urllib.request import urlopen from urllib.request import urlretrieve import json食物热量&营养元素爬虫
firstpage_url = "http://www.hpcn21.com/food"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36',
'Cookie': 'Hm_lvt_397c6669caf198f993b0bb244a74b53a=1657502938; Hm_lpvt_397c6669caf198f993b0bb244a74b53a=1657503671'}
#获取首页食物分类，生成分类页面网址
firstpage = requests.get(firstpage_url, headers=headers)
bs_firstpage = bs4.BeautifulSoup(firstpage.text, 'html.parser')
cate_list_url = []
for div in bs_firstpage.find_all('div',class_='cate'): #查找分类标签
cate = div.find('a')['href'].split('/')[-1] #提取分类
cate_list_url.append(firstpage_url + '/'+ cate) #构造网址
#逐个分类网址获取该分类下共有多少个page,生成每个page的网址
food_page_url = []
for url in cate_list_url:
food_cate= requests.get(url, headers=headers) #打开每一个分类的首页网址
bs_food_cate = bs4.BeautifulSoup(food_cate.text, 'html.parser') #
page_max_text = bs_food_cate.find('div',class_='pagelink').find_all('a')[-1] #获取尾页的页面链接
page_max = int(re.findall("d+",str(page_max_text))[0]) #提取页面最大值
sleep_time = np.random.randint(low=1,high=5)
time.sleep(sleep_time) #随机等待1~5秒
food_page_url = food_page_url + [url + '/p' + str(x) + '.html' for x in range(1,page_max + 1)] #生成所有页面的网址
爬取食物热量并下载食物图片
all_food_dict = {} #存储食物信息的空字典
for url in food_page_url: #遍历所有的食物page
# print('正在爬取:'+ url)
foodlist = requests.get(url, headers=headers) #打开每一个分类的首页网址
bs_foodlist= bs4.BeautifulSoup(foodlist.text, 'html.parser') #
food_cate = bs_foodlist.find('h1').text.split('：')[0] #获取食物大类
for food in bs_foodlist.find_all('div',class_="c fw_box_tt"): #遍历食物信息list，每个list 10个food
foodinfo_dict = {} #存储单个食物信息的字典
foodinfo_dict['foodcate'] = food_cate
food_id = food.find_all('p')[0].find('a')['href'].split('/')[-1] #食物id
foodinfo_dict['id'] = food_id
foodinfo_dict['name'] = food.find_all('p')[0].find('a')['title']
img_url = food.find_all('p')[0].find('img')['src']
foodinfo_dict['img'] = food.find_all('p')[0].find('img')['src']
# path = 'D:\Python\Jupyter\6 爬虫\01食物热量爬虫\图片库\' + img_url.split('/')[-1]
# urlretrieve(img_url,path) #下载图片到指定位置
foodinfo_dict['alias'] = food.find_all('p')[1].text #食物别名
foodinfo_dict['heat_cal'] = food.find_all('p')[2].text #食物热量
all_food_dict[food_id] = foodinfo_dict
# print('已爬取{}条信息'.format(len(all_food_dict.keys())))
# sleep_time = np.random.randint(low=1,high=5)
# time.sleep(sleep_time) #随机等待5-10秒
数据处理为dataframe格式
#数据处理
df = pd.DataFrame(all_food_dict.values())
df['alias'] = df['alias'].apply(lambda x : x.split('：')[1])
df['heat_cal'] = df['heat_cal'].apply(lambda x:re.search("d+(.d+)?",x).group()).astype('float') #正则匹配食物热量
食物营养元素明细爬虫
#食物详细营养元素爬虫
food_detail_dict = {}
for food_id in df['id'].values:
food_url = 'http://www.hpcn21.com/shiwu/' + food_id
food_detail = requests.get(food_url, headers=headers) #打开每一个分类的首页网址
bs_food_detail= bs4.BeautifulSoup(food_detail.text, 'html.parser') #
detail_name = bs_food_detail.find_all('span',class_='dt')
detail_num = bs_food_detail.find_all('span',class_='dd')
food_dt_dict = {}
for name,num in zip(detail_name,detail_num[1:]):
food_dt_dict[name.text] = num.text
food_detail_dict[food_id] = food_dt_dict
if (len(food_detail_dict.keys()) % 100) == 0: #每100条打印一次进度
print('已爬取{}条信息'.format(len(food_detail_dict.keys())))
sleep_time = np.random.randint(low=1,high=5)
time.sleep(sleep_time) #随机等待5-10秒
#数据合并
food_detail_df = pd.DataFrame(food_detail_dict.values())
food_detail_df['id'] = pd.DataFrame(food_detail_dict.keys())
final_food_detail = df.merge(food_detail_df,how='left',on='id') #合并数据
食谱数据爬虫
##爬取菜谱列表
caipu_url = 'http://www.hpcn21.com/caipu/2'
res = requests.get(caipu_url,headers=headers)
bs_res= bs4.BeautifulSoup(res.text, 'html.parser')
caipu_url_list = []
for href in bs_res.find('div',class_='caipu').find_all('a'): #菜系列表
caipu_url_list.append('http://www.hpcn21.com' + href['href'])
caipu_page_url = []
for url in caipu_url_list:
caipu= requests.get(url, headers=headers) #打开每一个分类的首页网址
bs_caipu = bs4.BeautifulSoup(caipu.text, 'html.parser') #
last_page = bs_caipu.find('div',class_='pagelink').find_all('a') #获取所有页面链接
if len(last_page) > 0: #个别菜系只有一页，获取不到尾页链接
page_max_text = bs_caipu.find('div',class_='pagelink').find_all('a')[-1] #获取尾页的页面链接
page_max = int(re.findall("d+",str(page_max_text['href'].split('/')[3]))[0]) #提取页面最大值
caipu_page_url = caipu_page_url + [url + '/p' + str(x) + '.html' for x in range(1,page_max + 1)] #生成所有页面的网址
else:
caipu_page_url = caipu_page_url + [url]
all_caipu_dict = {} #存储食物信息的空字典
for url in caipu_page_url: #遍历所有的食物page
caipulist = requests.get(url, headers=headers) #打开每一个分类的首页网址
bs_caipulist= bs4.BeautifulSoup(caipulist.text, 'html.parser') #
caipu_cate = bs_caipulist.find('h1').text#获取食物大类
for food in bs_caipulist.find_all('div',class_="c"):
foodinfo_dict = {} #存储单个食物信息的字典
foodinfo_dict['caixi'] = caipu_cate
food_id = food.find_all('p')[0].find('a')['href'].split('/')[-1] #食物id
foodinfo_dict['id'] = food_id
foodinfo_dict['name'] = food.find_all('p')[0].find('a')['title']
img_url = food.find_all('p')[0].find('img')['src']
foodinfo_dict['img'] = food.find_all('p')[0].find('img')['src']
# path = 'D:\Python\Jupyter\6 爬虫\01食物热量爬虫\食谱图片库\' + img_url.split('/')[-1]
# urlretrieve(img_url,path) #下载图片到指定位置
foodinfo_dict['type'] = food.find_all('p')[1].text #食物别名
foodinfo_dict['heat_cal'] = food.find_all('p')[2].text #食物热量
all_caipu_dict[food_id] = foodinfo_dict
# print('已爬取{}条信息'.format(len(all_food_dict.keys())))
# sleep_time = np.random.randint(low=1,high=5)
# time.sleep(sleep_time) #随机等待5-10秒
df_caipu = pd.DataFrame(all_caipu_dict.values())
df_caipu.head()
