腾讯云对象存储COS是类似于阿里云OSS，相比OSS，COS提供每月免费额度：存储空间50G、外网访问流量10G（内网免费）、免费读请求100万次、写请求10万次。对网站备份来说不错，但是，腾讯云提供的工具太low，参考阿里云OSS，写了一个cos信息配置、创建目录、上传、批量删除工具（coscmd）
安装Python2.7此工具在Python2.7上测试通过，建议用《OneinStack》安装Python2.7，安装路径为:/usr/local/python，命令如下：
cd ~/oneinstack
./addons.sh #选择7，install Let's Encrypt client
登陆腾讯云管理后台https://console.qcloud.com/cos/bucket创建bucket，并获取API密钥，在下面配置中一一对应。
cd oneinstack/tools #必须进入该目录执行
wget http://mirrors.linuxeye.com/oneinstack/tools/cos.tgz #v4版本
wget http://mirrors.linuxeye.com/oneinstack/tools/cos_v3.tgz #v3版本，可提交工单升v4
tar xzf cos.tgz
/usr/local/python/bin/python ./coscmd config --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket] #v4配置
/usr/local/python/bin/python ./coscmd config --appid=[appid] --id=[secret_id] --key=[secret_key] --bucket=[bucket] #v3配置
COSv4版本配置
COSv3版本配置
执行后会将相关信息写到~/.coscredentials，执行其它动作会自动加在改配置。
创建目录 上传文件 批量删除 coscmd代码(COSv4)#!/usr/bin/env python
#coding:utf-8
import sys,os
import datetime
import random
import threading
import time
import datetime
import logging
import ConfigParser
from optparse import OptionParser
from logging.handlers import RotatingFileHandler
from time import strftime, localtime
from time import sleep
from datetime import date
from datetime import timedelta
from cos import CosClient
from cos import UploadFileRequest
from cos import CreateFolderRequest
from cos import DelFileRequest
from cos import DelFolderRequest
from cos import ListFolderRequest
from cos import threadpool
MAX_RETRY_TIMES = 3
LOG_SAVE_EVERY_NUM = 1024
ONE_TASK_DEL_FILE_NUMS = 50
log_level = 1
log_file_name = "del_file.log"
dir_thread_num = 2
file_thread_num = 5
log_out_to_screen = 1
delete_folder_fail_exist = 0
CONFIGFILE = "%s/.coscredentials" % os.path.expanduser('~')
CONFIGSECTION = 'COSCredentials'
HAS_FORK = hasattr(os, 'fork')
HELP =
'''coscmd:
config --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket]
ls cosdir
mkdir dirname
put localfile cosdir
rm(delete,del) object
'''
CMD_LIST = {}
def cmd_configure(args, options):
if options.appid is None or options.secret_id is None or options.secret_key is None or options.region is None or options.bucket is None:
print("%s miss parameters, use --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket] to specify appid/id/key/region/bucket pair" % args[0])
sys.exit(-1)
config = ConfigParser.RawConfigParser()
config.add_section(CONFIGSECTION)
config.set(CONFIGSECTION, 'appid', options.appid)
config.set(CONFIGSECTION, 'secret_id', options.secret_id)
config.set(CONFIGSECTION, 'secret_key', options.secret_key)
if options.region in ['sh','gz','tj','sgp']:
config.set(CONFIGSECTION, 'region', options.region)
else:
print("input region error, setup use : --region={sh,gz,tj,sgp}")
sys.exit(-1)
config.set(CONFIGSECTION, 'bucket', options.bucket)
cfgfile = open(CONFIGFILE, 'w+')
config.write(cfgfile)
print("Your configuration is saved into %s ." % CONFIGFILE)
cfgfile.close()
import stat
os.chmod(CONFIGFILE, stat.S_IREAD | stat.S_IWRITE)
def cmd_loadconfigure():
config = ConfigParser.ConfigParser()
config.read(CONFIGFILE)
global appid
global secret_id
global secret_key
global region
global bucket
appid = int(config.get(CONFIGSECTION, 'appid'))
secret_id = config.get(CONFIGSECTION, 'secret_id').decode('utf-8')
secret_key = config.get(CONFIGSECTION, 'secret_key').decode('utf-8')
region = config.get(CONFIGSECTION, 'region')
bucket = config.get(CONFIGSECTION, 'bucket').decode('utf-8')
if len(secret_id) == 0 or len(secret_key) == 0 or len(region) == 0 or len(bucket) == 0:
print("can't get appid/secret_id/secret_key/region/bucket, setup use : config --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket]")
sys.exit(1)
def cmd_lsdir(COSDIR):
cosdir = COSDIR.decode('utf-8')
request = ListFolderRequest(bucket, cosdir)
list_folder_ret = cos_client.list_folder(request)
if list_folder_ret[u'code'] == 0:
print(True)
else:
print("%s, appid/secret_id/secret_key/region/bucket invalid"% list_folder_ret[u'message'])
def cmd_mkdir(COSDIR):
cosdir = COSDIR.decode('utf-8')
request = CreateFolderRequest(bucket, cosdir)
create_folder_ret = cos_client.create_folder(request)
if create_folder_ret[u'code'] == 0:
print("mkdir cos://%s%s OK" % (bucket,COSDIR))
else:
print(create_folder_ret[u'message'])
def cmd_put(LOCALFILE,COSFILE):
localfile = LOCALFILE.decode('utf-8')
cosfile = COSFILE.decode('utf-8')
request = UploadFileRequest(bucket, cosfile, localfile)
request.set_insert_only(0)
upload_file_ret = cos_client.upload_file(request)
if upload_file_ret[u'code'] == 0:
print("put cos://%s%s OK" % (bucket,COSFILE))
else:
print(upload_file_ret[u'message'])
def loginit():
global config
if (log_file_name == ""):
return
log_level = logging.ERROR
if log_level == 0:
log_level = logging.DEBUG
if log_level == 1:
log_level = logging.INFO
if log_level == 2:
log_level = logging.WARNING
#定义一个RotatingFileHandler，最多备份5个日志文件，每个日志文件最大20M
logger = logging.getLogger("")
Rthandler = RotatingFileHandler(log_file_name, maxBytes= 20*1024*1024,backupCount=5)
Rthandler.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
Rthandler.setFormatter(formatter)
logger.addHandler(Rthandler)
#输出日志到屏幕
console = logging.StreamHandler()
console.setFormatter(formatter)
if (log_out_to_screen == 1):
logger.addHandler(console)
logger.setLevel(log_level)
return logger
#日期相关操作
class Dateop():
@staticmethod
def isValidDate(str):
try:
time.strptime(str, "%Y""%m""%d")
return True
except:
return False
@staticmethod
def getdaystr(n=0):
dt = date.today()-timedelta(days=n)
tt = dt.timetuple()
daystr = strftime("%Y""%m""%d",tt)
return daystr
@staticmethod
def cmpDateAgo(t1,t2):
if (Dateop.isValidDate(t1)==False or Dateop.isValidDate(t2)==False):
return False
if (int(t1) <= int (t2)):
return True
return False
@staticmethod
def isNeedDeleteDir(dirname, n=0):
if (len(dirname) != 8):
return False
if Dateop.isValidDate(dirname) == False:
return False
d2 = Dateop.getdaystr(n);
if Dateop.cmpDateAgo(dirname, d2):
return True
return False
#删除文件统计
class FileStat():
global cos_log
def __init__(self):
self.delfilesuccnum = 0
self.deldirsuccnum = 0
self.delfilefailnum = 0
self.deldirfailnum = 0
self.lock = threading.Lock()
def addDelFileFailNum(self,num=1):
self.lock.acquire(1)
self.delfilefailnum += num
self.lock.release()
def addDelDirFailNum(self,num=1):
self.lock.acquire(1)
self.deldirfailnum += num
self.lock.release()
def addDelDirSuccNum(self, num=1):
self.lock.acquire(1)
self.deldirsuccnum += num
self.lock.release()
def addDelFileSuccNum(self, num=1):
self.lock.acquire(1)
self.delfilesuccnum += num
self.lock.release()
def printStat(self):
msg ="".join(["delfilesuccnum=",str(self.delfilesuccnum),
",delfilefailnum=",str(self.delfilefailnum),
",deldirsuccnum=",str(self.deldirsuccnum),
",deldirfailnum=",str(self.deldirfailnum)])
print(msg)
def logStat(self):
curtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
log = ''.join(["delfilenum=",str(self.delfilesuccnum),
",deldirnum=",str(self.deldirsuccnum),",delfilefailnum=",
str(self.delfilefailnum),",deldirfailnum=",str(self.deldirfailnum)])
cos_log.info(log)
#执行时间统计
class TimeStat(object):
global cos_log
def __init__(self):
self.start()
def start(self):
self.start = datetime.datetime.now()
self.t1 = time.time()
msg = "delete task started ..........."
cos_log.info(msg)
def end(self):
self.end = datetime.datetime.now()
self.t2 = time.time()
msg = "delete task endednnrm task finished,ntimecost:"+str(self.t2-self.t1) + " (s)"
cos_log.info(msg)
#删除文件列表中的文件
def delfiles(cos_client, bucket, filelist):
for f in filelist:
delfile(cos_client, bucket, f)
def delfolders(cos_client, bucket, folderlist):
for f in folderlist:
delfolder(cos_client, bucket, f)
#文件夹删除
def delfolder(cos_client, bucket, folder):
global stat
global cos_log
if not folder:
return 0
delfolderreq = DelFolderRequest(bucket, folder)
retry = 0
while (retry < MAX_RETRY_TIMES):
ret = cos_client.del_folder(delfolderreq)
msg = "delfolder fail, bucket="+bucket+",folder="+folder+ret['message']
if (ret['code'] == 0):
break
elif (ret['code'] == -166):
cos_log.warning(msg)
break
#操作太频繁,频控
elif (ret['code'] == -71):
sleep(random.randint(1,5))
cos_log.warning(msg)
retry += 1
continue
#文件夹非空
elif (ret['code'] == -173):
break
else:
cos_log.warning(msg)
retry += 1
if (ret['code'] != 0 and ret['code'] != -166):
stat.addDelDirFailNum()
cos_log.error("delfolder fail, bucket="+bucket+",folder="+folder+ret['message'])
return ret['code']
if (ret['code'] == 0):
stat.addDelDirSuccNum()
msg = "delfolder success, bucket="+bucket+",folder="+folder
cos_log.info(msg)
return 0
#文件删除
def delfile(cos_client, bucket, filepath):
global stat
global cos_log
delfilereq = DelFileRequest(bucket, filepath)
retry = 0
while (retry < MAX_RETRY_TIMES):
ret = cos_client.del_file(delfilereq)
msg = "delfile fail bucket="+bucket+",file="+filepath+ret['message']
if (ret['code'] == 0):
break
#文件不存在
elif (ret['code'] == -166):
cos_log.warning(msg)
break
#单目录写操作过快
elif (ret['code'] == -143):
sleep(random.randint(1,5))
cos_log.warning(msg)
retry += 1
continue
#操作太频繁,频控
elif (ret['code'] == -71):
sleep(random.randint(1,5))
cos_log.warning(msg)
retry += 1
continue
else:
cos_log.warning(msg)
retry += 1
continue
if (ret['code'] != 0 and ret['code'] != -166):
stat.addDelFileFailNum()
cos_log.error("delfile fail, bucket="+bucket+",file="+filepath+ret['message'])
return ret['code']
if (ret['code'] == 0):
stat.addDelFileSuccNum()
msg = "delfile success, bucket="+bucket+",file="+filepath
cos_log.info(msg)
return 0
#递归文件夹进行文件删除
def delete_r(cos_client, bucket, path, thread_pool_file):
global stat
global config
global cos_log
cos_log.debug("delete_r bucket:"+bucket+",path:"+path)
context = u""
#递归文件夹
while True:
listfolderreq = ListFolderRequest(bucket, path, 1000, u'', context)
retry = 0
while (retry < MAX_RETRY_TIMES):
listret = cos_client.list_folder(listfolderreq)
if listret['code'] != 0 :
retry += 1
sleep(random.randint(1,3))
continue
else:
break
if (listret['code'] != 0):
cos_log.error("delete_r: list folder fail:"+path +",return msg:"+ listret['message'])
return listret['code']
if (len(listret['data']['infos']) == 0):
break;
filelist = []
dirlist = []
for info in listret['data']['infos']:
fullname = path + info['name']
#list出来的文件列表中文件夹和文件本身是混杂一起的
if info.has_key('filesize'):
filelist.append(fullname)
if (len(filelist) >= ONE_TASK_DEL_FILE_NUMS):
args = [cos_client, bucket, filelist]
args_tuple = (args,None)
args_list = [args_tuple]
requests = threadpool.makeRequests(delfiles, args_list)
for req in requests:
thread_pool_file.putRequest(req)
filelist = []
continue
else:
pass
else:
dirlist.append(fullname)
if (len(dirlist) >= ONE_TASK_DEL_FILE_NUMS):
args = [cos_client, bucket, dirlist]
args_tuple = (args,None)
args_list = [args_tuple]
requests = threadpool.makeRequests(delfolders, args_list)
for req in requests:
thread_pool_file.putRequest(req)
dirlist = []
continue
else:
pass
pass
if (len(filelist) > 0):
args = [cos_client, bucket, filelist]
args_tuple = (args,None)
args_list = [args_tuple]
requests = threadpool.makeRequests(delfiles, args_list)
for req in requests:
thread_pool_file.putRequest(req)
filelist = []
else:
pass
if (len(dirlist) > 0):
args = [cos_client, bucket, dirlist]
args_tuple = (args,None)
args_list = [args_tuple]
requests = threadpool.makeRequests(delfolders, args_list)
for req in requests:
thread_pool_file.putRequest(req)
filelist = []
else:
pass
cos_log.debug("delete_r thread pool file waitingn")
thread_pool_file.wait()
cos_log.debug("delete_r thread pool file waiting endn")
if (listret['data']['listover'] == False):
context = listret['data']['context']
continue
else:
break
stat.logStat()
return 0
#支持Ctrl+C终止程序
class Watcher():
def __init__(self):
self.child = os.fork()
if self.child == 0:
return
else:
self.watch()
def watch(self):
global cos_log
try:
os.wait()
except KeyboardInterrupt:
cos_log.ERROR("ctrl+c terminated rm_recursive.py, exiting...")
self.kill()
sys.exit()
def kill(self):
try:
os.kill(self.child, signal.SIGKILL)
except OSError:
pass
def cmd_rm(COSDIR):
global thread_pool
global cos_log
global stat
cos_log = loginit()
stat = FileStat()
timestat = TimeStat()
if HAS_FORK:
Watcher()
path = COSDIR.decode('utf-8')
thread_pool_dir = threadpool.ThreadPool(dir_thread_num)
thread_pool_file = threadpool.ThreadPool(file_thread_num)
cos_log.debug("bucket:"+bucket +",path:"+path)
args = [cos_client, bucket, path, thread_pool_file]
args_tuple = (args, None)
args_list = [args_tuple]
requests = threadpool.makeRequests(delete_r, args_list)
for req in requests:
thread_pool_dir.putRequest(req)
cos_log.debug("thread_pool_dir waiting.....n")
thread_pool_dir.wait()
thread_pool_dir.dismissWorkers(dir_thread_num, True)
cos_log.debug("thread_pool_dir wait end.....n")
timestat.end()
stat.logStat()
if sys.argv[1] in ['config','ls','mkdir','put','rm','delete','del'] and len(sys.argv) >= 3:
if sys.argv[1] == 'config':
parser = OptionParser()
parser.add_option("-a", "--appid", dest="appid", help="specify appid")
parser.add_option("-i", "--id", dest="secret_id", help="specify secret id")
parser.add_option("-k", "--key", dest="secret_key", help="specify secret key")
parser.add_option("-r", "--region", dest="region", help="specify region")
parser.add_option("-b", "--bucket", dest="bucket", help="specify bucket")
(options, args) = parser.parse_args()
CMD_LIST['config'] = cmd_configure
CMD_LIST['config'](args, options)
if sys.argv[1] == 'ls':
cmd_loadconfigure()
cos_client = CosClient(appid, secret_id, secret_key, region)
COSDIR = sys.argv[2]
cmd_lsdir(COSDIR)
if sys.argv[1] == 'mkdir':
cmd_loadconfigure()
cos_client = CosClient(appid, secret_id, secret_key, region)
COSDIR = sys.argv[2]
cmd_mkdir(COSDIR)
if sys.argv[1] == 'put' and len(sys.argv) == 4:
cmd_loadconfigure()
cos_client = CosClient(appid, secret_id, secret_key, region)
LOCALFILE = sys.argv[2]
COSFILE = sys.argv[3]
cmd_put(LOCALFILE,COSFILE)
if sys.argv[1] in ('rm','delete','del'):
cmd_loadconfigure()
cos_client = CosClient(appid, secret_id, secret_key, region)
COSDIR = sys.argv[2]
path = COSDIR.decode('utf-8')
cmd_rm(path)
else:
print(HELP)
exit()
Fri Mar 31 21:51:20 CST 2017
