vx-underground样本快速获取方法

vx-underground是个不错的开源样本平台。最近需要下载好几个开源样本平台的样本，对python爬虫研究比较多，拿这个来记录下。

总而言之吧，自己写的python request程序有时候会被服务器限制，现在很多成熟的商业软件。如果能获取到下载链接，直接塞迅雷这种下载软件里比较迅速稳定，自己写代码没必要。另外可以配合八爪鱼一些轻松获取下载链接的爬虫工具。

import requests
import re
import os
import random
import time

requests.adapters.DEFAULT_RETRIES = 5
user_agent_list = ["Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
                    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
                    "Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/61.0",
                    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
                    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
                    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15",
                ] 
                
agent = random.choice(user_agent_list)                
headers = {
    'Upgrade-Insecure-Requests': '1',
    'Connection': 'close',
    'User-Agent': agent
 } 
 
url = 'https://samples.vx-underground.org/samples/Families/'   

def get_sample_family():
       
    global headers,url      
    r = requests.get(url,headers=headers)
    with open("./html1.txt", "wb") as handle:
        handle.write(r.content)
        handle.close()
    regex = re.compile('(?<=title=").*?(?=")')
    viruslist = regex.findall(str(r.content))
    with open("./virusfamily.txt", "a") as handle:
        for temp in viruslist:
            name = temp.split('"')
            virusname = name[-1]
            print(virusname)
            handle.write(virusname)
            handle.write("n")
    handle.close()
    
def get_sample_download_urls():
     
    global headers,url
    with open("./virusfamily.txt", "r") as handle:
        lines = handle.readlines()
        handle.close()
    for virusname in lines:
        hashurl = url + virusname.strip() 
        print(hashurl)
        r = requests.get(hashurl,headers=headers)
        time.sleep(3)
        if str(r.content).find('title="Papers"') and str(r.content).find('title="Samples"')!=-1:
            hashurl = url + virusname.strip() + "/Samples"
            print(hashurl)
            r = requests.get(hashurl,headers=headers)
            time.sleep(3)
        with open("./html2.txt", "a") as handle:
            handle.write(str(r.content))
            handle.close()
        regex = re.compile('(?<=href=").*?(?=")')
        downloadlist = regex.findall(str(r.content))
        with open("./download_urls.txt", "a") as handle:
            for temp in downloadlist:
                print(temp)
                handle.write(temp)
                handle.write("n")
        handle.close()
    
    
def main():
 
    if not os.path.exists("./virusfamily.txt"):
        print("1    test connceting to vx-underground......")
        get_sample_family()
        print("get virus family success!")
    else:
        print("virusfamily exist!next step!")
    if not os.path.exists("./download_urls.txt"): 
        print("2    test getting sample download urls......")
        get_sample_download_urls()
    else:
        print("download urls exist!next step!")
    
if __name__=="__main__":
    main()

import zipfile
import py7zr
import os

def uncompress(path_name):
    print(path_name)
    suffix = path_name.rsplit('.', 1)[1]
    if suffix == 'zip':
        if zipfile.is_zipfile(path_name):
            try:
                with zipfile.ZipFile(path_name) as zip_f:
                    zip_f.extractall(path_name.rsplit(".zip")[0])
            except Exception as e:
                print('Error when uncompress file! info: ', e)
                return False
            else:
                return True
        else:
            print('This is not a true zip file!')
            return False
    if suffix == '7z':
        if py7zr.is_7zfile(path_name):
            try:
                d_name = 'infected'
                with py7zr.SevenZipFile(path_name,password=d_name, mode='r') as sevenZ_f:
                    sevenZ_f.extractall(path_name.rsplit(".7z")[0])
            except Exception as e:
                print('Error when uncompress file! info: ', e)
                return False
            else:
                return True
        else:
            print('This is not a true 7z file!')
            return False


if __name__ == '__main__':
    folder_name = input("please input folder--for example--E:\workspace\unzipTest：")
    count = 0
    os.chdir(folder_name)
    files = os.listdir(folder_name)
    for f in files:
        f_path = folder_name + os.sep + f
        if os.path.isfile(f_path):
            print("解压--"+f)
            uncompress(path_name=f_path)
            os.remove(f_path)
            count = count + 1
            print(count)