批量转化GBK编码至UTF-8编码工具【Python】

github地址

pip install utf8conv

github 发布

# 帮助
python -m utf8conv -h

# 基本使用
python -m utf8conv -i 输入文件/目录 -o 输出目录 -t 格式列表

# 使用示例,转化code 目录下 .txt .json .c .cpp .py 为utf-8 格式
python -m utf8conv 
	-i /root/code 
	-o /root/code_u8 
	-s .txt .json .c .cpp .py
	
	
# 转化文本文件
python -m utf8conv -i /root/code/abc.cpp -o ./

import argparse
from pathlib import Path
import chardet
from tqdm import tqdm
import shutil
from time import time

parser = argparse.ArgumentParser(description='convert file to utf-8 encoding')
parser.add_argument('-i', '--input', help='input a folder or a single_file')
parser.add_argument('-o', '--output_folder', help='output folder for converted files')
parser.add_argument('-s', '--suffix', nargs='+', help='file type list. eg.: .txt .c .json .py .', default=['.txt'])
parser.add_argument('-v', '--verbose', action='store_true', help='show details')
args = parser.parse_args()

st = time()

file_suffix = set([t.lower() for t in args.suffix])

# get file list
proc_file_list = [Path(args.input).resolve()] 
    if Path(args.input).is_file() 
    else [file.resolve() for file in Path(args.input).rglob('*') if file.suffix.lower() in file_suffix]

# to maintain the tree structure, get the max same depth of input folder
max_depth = len(str(proc_file_list[0].parent)) + 1 
    if len(proc_file_list) == 1 
    else len(str(max(set.intersection(*map(set, [[p for p in f.parents] for f in proc_file_list]))))) + 1

num_file = len(proc_file_list)
if not args.verbose:
    proc_file_list = tqdm(proc_file_list)

for n, f in enumerate(proc_file_list):
    target_path = Path(args.output_folder) / str(f)[max_depth:]
    if not target_path.parent.is_dir():
        target_path.parent.mkdir(parents=True)
    with open(f, 'rb') as file:
        # using chardet lib to detect text encoding
        det = chardet.detect(file.read())
        det_encoding = det['encoding'].upper() if det['encoding'] is not None else 'UTF-8'
    coding = 'GBK' if 'GB' in det_encoding else det_encoding

    # cov other encoding text into utf-8, copy utf-8 to target folder
    if coding != 'UTF-8':
        with open(f, 'r', encoding=coding) as file:
            txt = file.read()
        with open(target_path, 'w+', encoding='utf-8') as file:
            file.write(txt)
    else:
        shutil.copyfile(f, target_path)

    if args.verbose:
        print(' %.2f%% - [%i/%i] - %s - saved file <%s> to <%s>' %
              ((n + 1) / num_file * 100,
               n + 1, num_file,
               det_encoding,
               str(f.resolve()),
               str((Path(args.output_folder) / str(f)[max_depth:]).resolve())
               ))
if args.verbose:
    print('========================================')
    print('spend time: %s s' % (time()-st))