2021-09-27

刚进组老板就让我做关于高峰频耀变体的工作，目前高能这一块费米卫星是重要数据来源

FSSC: Data » Data Access » LAT (nasa.gov)https://fermi.gsfc.nasa.gov/ssc/data/access/lat/msl_lc/

这个网页是NASA下面的，包含费米卫星一直保持监测的一百多个源的fits数据，竟然没有反爬，header都无需设置，多进程必须安排上，代码如下

import requests
from lxml import etree
from multiprocessing import Pool
def get_html(url):
    response=requests.get(url)
    html=etree.HTML(response.content)
    return html
def get_fits_url(url):
    html=get_html(url)
    url_name=html.xpath('/html/body/div[2]/div[4]/div[2]/table//tr/td[1]')
    fits_url_name=[]
    for i  in url_name:
        fitsurlname=[]
        if i.xpath('./ul/li[3]/a/@href')!=[]:
            fitsurlname.append('https://fermi.gsfc.nasa.gov'+i.xpath('./ul/li[3]/a/@href')[0])
            fitsurlname.append(i.xpath('./a/text()')[0])
            fits_url_name.append(fitsurlname)
    return fits_url_name
def download_fits(fitsurl,fitsname):
    response=requests.get(fitsurl)
    with open(fitsname+'.fits','wb') as file:
        file.write(response.content)
    print(fitsname+'++++++++++++++++++++下载完成')
if __name__=='__main__':
    url='https://fermi.gsfc.nasa.gov/ssc/data/access/lat/msl_lc/'
    fits_url_name=get_fits_url(url)
    pool=Pool(10)  #进程数最好不要超过CPU核心数的10倍
    for i in fits_url_name:
        pool.apply_async(download_fits, args=(i[0],i[1],))
    pool.close()
    pool.join()