包括了：
- 使用pungi生成isoftp服务器使用python互联上传文件生成MD5值mock环境、配置文件，外部环境复制到mock环境中、在mock环境中执行命令，mock环境的优点不在叙述。
import pprint
import os
import shutil
import configparser
import logging
import re
import time
import subprocess
import pwd
import sys
import yaml
# import pyjsonrpc
import rpm
def read_rpm_header(rpmfile): # 获取rpm的header，rpmfile是rpm包的路径加名字如：/root/wqz/gnutls-3.6.14-7.p01.ky10.x86_64.rpm
ts = rpm.TransactionSet()
ts.setVSFlags(-1)
fd = os.open(rpmfile, os.O_RDONLY)
h = ts.hdrFromFdno(fd)
os.close(fd)
return h #
def check_rpm_ok(rpmfile): # 检查rpm
fd = None
try:
ts = rpm.TransactionSet()
ts.setVSFlags(-1)
fd = os.open(rpmfile, os.O_RDONLY)
h = ts.hdrFromFdno(fd)
return True
except Exception as e:
return False
finally:
if fd != None:
os.close(fd)
def get_rpm_nvr(rpmfile): # 获取rpm的nvr
h = read_rpm_header(rpmfile)
# nvr = ('-').join([h['name'], h['version'], h['release']]) # gnutls-3.6.14-7.p01.ky10
return h['Signature']
def run_command(cmd): # 创建子进程
p = subprocess.Popen(cmd, shell=True)
p.wait()
def set_user(name): # 用户相关
uid = pwd.getpwnam(name)[2]
gid = pwd.getpwnam(name)[3]
os.setgid(gid)
os.setuid(uid)
def change_user(name): # 用户相关
def _change_user(func):
def _change(*args, **kargs):
cur_gid = os.getgid()
cur_uid = os.getuid()
set_user(name)
func(*args, **kargs)
os.setgid(cur_gid)
os.setuid(cur_uid)
return _change
return _change_user
def _get_args():
import argparse
parser = argparse.ArgumentParser(description="The tools to release os iso.")
parser.add_argument("-c", "--config",
help="Please provide config file") # python python脚本-1.py -c haha 或者python python脚本-1.py --config haha 这个后面跟的就是配置文件名称
args = parser.parse_args()
if args.config:
return args.config
else:
sys.exit(1)
class ReleaseOS():
def _get_value(self):
"""
读取配置文件信息
:return:
"""
config = configparser.RawConfigParser(allow_no_value=True)
config_file = _get_args()
if os.path.exists(config_file):
config.read(config_file)
else:
print('Can not found %s file' % config_file)
sys.exit(1)
# self.mock_conf_name = config.get('mock','mock_conf_name') #mock config file just name ,not include suffix
self.ftp_ip = config.get('ftp', 'ip')
self.ftp_user = config.get('ftp', 'user')
self.ftp_passwd = config.get('ftp', 'password')
self.ftp_upload_dir = config.get('ftp', 'upload_dir')
self.pungi_user = config.get('mock', 'user')
# self.boot_dir = config.get('info', 'boot_dir')
# self.target_dir = config.get('info','target_dir')
self.product_name = config.get('info', 'product_name')
# temp vaule, for pylorax and pypungi, add by huachao
# self.newpungi_file_path = config.get('info', 'newpungi_file_path')
# self.newpungi_init_path = config.get('info', 'newpungi_init_path')
# self.newpylorax_init_path = config.get('info', 'newpylorax_init_path')
# self.newpylorax_treebuilder_path = config.get('info', 'newpylorax_treebuilder_path')
# self.newpylorax_ltmpl_path = config.get('info', 'newpylorax_ltmpl_path')
# end by huachao
self.tag = config.get('info', 'tag')
self.target_arch = config.get('info', 'target_arch')
self.type = config.get('info', 'type')
self.build = config.get('info', 'build')
self.build_num = self.build.split('-')[1].split('Build')[1]
if config.has_option('info', 'branch'):
self.branch = config.get('info', 'branch')
self.branch_num = re.findall(r'd+', self.branch)[0]
else:
self.branch = None
self.product_info = config.get('info', 'product_info')
# self.before_repo_package = config.get('info', 'before_repo_package')
# self.after_repo_package = config.get('info', 'after_repo_package')
# self.other_package = config.get('info','other_package')
# mash_output_dir = 'ns7.0/os'
self.release = config.get('info', 'release') # RELEASE="Build20"
self.kylin_release = config.get('info', 'kylin_release')
# self.comsxml = config.get('info', 'compsxml')
self.download_url = config.get('info', 'download_url')
# self.inherit = config.get('mash', 'inherit')
# self.output_dir = config.get('info', 'output_dir')
# self.release_dir = config.get('info', 'release_dir')
# self.notboot_dir = config.get('info', 'not_boot_dir')
self.local_iso_dir = os.path.join(config.get('info', 'local_iso_dir'), self.tag)
# self.mashed_output_dir = os.path.join(self.output_dir, 'mashed', self.tag, self.type)
self.target_dir = os.path.join(self.release, self.target_arch, self.type)
self.log_dir = os.path.join(config.get('info', 'log_dir'), self.tag)
self.mock_conf_name = ('-').join([self.tag, self.target_arch]) # mock配置文件的名字，需要在/etc/mock下
self.mock_build_root = os.path.join('/var/lib/mock', self.mock_conf_name, 'root')
self.upload_iso_dir = os.path.join(self.local_iso_dir, self.cur_time)
# add by wqz
self.workdir = os.getcwd()
self.local_iso_name = config.get('info', 'local_iso_name') # 制作yum源的iso名字
self.mock_repository = config.get('mock', 'local_repository')
self.version = config.get('info', 'version')
self.version1 = config.get('info', 'version1')
self.iso_arch = config.get('info', 'iso_arch')
self.mount_name = config.get('info', 'mount_name')
self.iso_name = self.mount_name + "-SP1-" + self.iso_arch + "-Release-" + self.release + self.cur_time + ".iso" # ${MOUNT_NAME}-SP1-${ISO_ARCH}-Release-${RELEASE}-${now_time}.iso
self.pungi_dir = config.get('mock', 'pungidir')
self.new_packages = config.get('info', 'new_packages')
self.compsxml_name = config.get('info', 'compsxml_name')
self.ks_file_name = config.get('info', 'ks_file_name')
self.compsxml = os.path.join(self.workdir, self.compsxml_name)
self.ks_file = os.path.join(self.pungi_dir, self.ks_file_name) # 这个是mock相对下的路径
self.local_repository = os.path.join(self.mock_build_root, self.mock_repository)
self.iso_path = "%s/iso/%s/%s/%s" % (
os.path.join(self.mock_build_root, self.pungi_dir), self.version1, self.release, self.mount_name)
# end by wqz
# _release_dir = os.path.join(self.release_dir, self.tag, self.type)
# todo
for _dir in [self.log_dir, self.local_iso_dir]:
if not os.path.exists(_dir):
os.makedirs(_dir)
def __init__(self):
"""
tag : koji tag
type: os or updates
release: release version
inhert: True or False, is not inhert from other tag
"""
self.cur_time = time.strftime('%Y%m%d%H', time.localtime(time.time()))
self.cur_year = time.strftime('%Y', time.localtime(time.time()))
self.cur_day = time.strftime('%Y%m%d', time.localtime(time.time()))
self.cur_mon = time.strftime('%m', time.localtime(time.time()))
self._get_value()
self.log_file = os.path.join(self.log_dir, '%s_%s.log' % (self.build, self.cur_time))
self.logger = logging.getLogger()
handler = logging.FileHandler(self.log_file)
self.logger.addHandler(handler)
self.logger.setLevel(logging.DEBUG)
def mock_copy(self, source, target):
"""
把外面的文件复制到mock环境里
:param source: 需要复制的资源
:param target: 需要复制到的地方
:return:
"""
cmd = 'mock -r %s --copyin %s %s' % (self.mock_conf_name, source, target)
runcmd = "su - %s -c '%s'" % (self.pungi_user, cmd)
return run_command(runcmd)
def copy_file_to_mock(self, src_dir, target_dir):
"""
Copy file to mock env.
:param src_dir: 需要复制的整个目录
:param target_dir: 需要复制到mock环境中的目录
:return:
"""
if os.path.exists(src_dir):
copy_files = os.listdir(src_dir)
if len(copy_files) > 0:
for f in copy_files:
self.mock_copy(os.path.join(src_dir, f), os.path.join(target_dir, f))
def create_pungi_mock_file(self):
"""
创建mock的配置文件
:return:None
"""
self.logger.info('生成mock配置文件----start')
cmd1 = "sed '$s/"""/[epel]/' /etc/mock/templates/centos-stream-8.tpl > /etc/mock/templates/%s.tpl" % (
self.mock_conf_name)
cmd2 = "echo '#baseurl=https://download.example/pub/epel/$releasever/Everything/$basearchnmetalink=https://mirrors.fedoraproject.org/metalink?repo=epel-$releasever&arch=$basearch&infra=$infra&contenn"""' >> /etc/mock/templates/%s.tpl" % (
self.mock_conf_name)
for cmd in [cmd1, cmd2]:
run_command(cmd)
content = '''include('templates/%s.tpl')
config_opts['root'] = '%s'
config_opts['target_arch'] = '%s'
config_opts['legal_host_arches'] = ('%s',)
''' % (self.mock_conf_name, self.mock_conf_name, self.target_arch, self.target_arch)
file = '/etc/mock/%s.cfg' % self.mock_conf_name
with open(file, 'w') as f:
if os.path.exists(file):
shutil.copy(file, '%s.bak' % file)
f.writelines(content)
self.logger.info('生成mock配置文件%s.cfg和%s.tpl完成---end' % (self.mock_conf_name, self.mock_conf_name))
def init_pungi_mock_env(self):
"""
生成所需的mock环境
:return:None
"""
self.logger.info('创建所需的mock环境---start')
build_root = os.path.join('/var/lib/mock', self.mock_conf_name)
self.logger.info('如果存在旧的mock目录旧则删除')
if os.path.exists(build_root):
shutil.rmtree(build_root)
self.logger.info('Delete build_root %s' % build_root)
# 1.初始化一个mock环境
self.logger.info('初始化mock环境')
cmd1 = 'su - %s -c "mock -n -r %s --init"' % (
self.pungi_user, self.mock_conf_name) # todo 现在为了测试，后期去掉-n
run_command(cmd1)
# 2.下载pungi和lorax
self.logger.info('下载pungi和lorax')
cmd2 = 'echo "----install----";su - %s -c "mock -n -r %s --install pungi lorax"' % (
self.pungi_user, self.mock_conf_name)
run_command(cmd2)
# 3.创建工作目录
self.logger.info('创建工作目录')
cmd3 = "mkdir -p %s/%s/tmp" % (self.mock_build_root, self.pungi_dir)
# 4.删除/usr/share/lorax/templates.d%define build_id ' + '%s/" $SPEC_FILE' % self.build_num
else:
cmd3 = 'sed -i "s/%define build_id.*/%define build_id ' + '%s.lic/" $SPEC_FILE' % self.build_num
cmd4 = 'sed -i "s/%define build_branch.*/%define build_branch ' + '%s/" $SPEC_FILE' % self.branch
cmd5 = 'sed -i "s/%define build_date.*/%define build_date ' + '%s/" $SPEC_FILE' % self.cur_day
# 在centos上编译不通过增加
cmd6 = 'sed -i "s/python/python3/" $SPEC_FILE'
cmd7 = 'sed -i "s/Thu Jul 03 2020 Shi Jingying/Fri Jul 03 2020 Shi Jingying/" $SPEC_FILE'
cmd8 = 'sed -i "s/Mon Feb 21 2021 Shi Jingying/Sun Feb 21 2021 Shi Jingying/" $SPEC_FILE'
cmd9 = 'sed -i "s/Fri Mar 19 2021 yangxudong/Fri Mar 10 2021/" $SPEC_FILE'
cmd10 = 'sed -i "s/python.*/python3/" $SPEC_FILE'
# %{ns_dist} Centos没有这个宏定义，需要增加,直接把咱们的宏定义放到这里
cmd11 = "echo '%dist .ky10n%ns_dist .pn%kylin 2n%_vendor kylin' >> /etc/rpm/macros.dist"
cmd12 = 'rpmbuild -ba $SPEC_FILE'
cmd13 = 'find %s|grep kylin-release|xargs rm -rvf' % package_dir
cmd14 = 'cp -rvf `find builddir/build/RPMS/ -name "*kylin-release*" ! -name "*debuginfo*"` %s' % package_dir
# bash-4.4# rpmbuild -ba /builddir/build/SPECS/kylin-release.spec
# cmd15 = 'cp -rvf `find builddir/build/SRPMS/ -name "*kylin-release*" ! -name "*src.rpm*"` %s' % package_dir
cmd15 = 'cp -rvf `find builddir/build/SRPMS/ -name "*kylin-release*"` /' # 没作用
for cmd in [cmd1, cmd2, cmd3, cmd4, cmd5, cmd6, cmd7, cmd8, cmd9, cmd10, cmd11, cmd12, cmd13, cmd14, cmd15]:
f.write(cmd)
f.write(os.linesep)
self.mock_copy(tmp_run_script, '/%s' % run_script)
_cmd = 'mock -r %s --chroot "sh %s" ' % (self.mock_conf_name, run_script)
_cmd = "su - %s -c '%s'" % (self.pungi_user, _cmd)
run_command(_cmd)
def replacement_package(self):
"""
版本替换，在replacement_package.yaml文件中定义需要替换的包名，或者需要增加的包名。
:return:
"""
package_dir = '/%s/os/Packages/' % (self.pungi_dir)
current_path = os.path.abspath(".")
yaml_file = os.path.join(current_path, "replacement_package.yaml")
file = open(yaml_file, 'r', encoding="utf-8")
file_data = file.read()
file.close()
datas = yaml.load(file_data)
for old_tar_name, new_tar_name in datas.items():
if old_tar_name == "add":
if new_tar_name:
for tar in new_tar_name:
self.mock_copy(os.path.join(self.new_packages, tar),
os.path.join(self.pungi_dir, 'os/Packages/'))
else:
_cmd1 = 'find %s|grep %s|xargs rm -rvf' % (package_dir, old_tar_name)
cmd1 = "su - %s -c 'mock -n -r %s --chroot "%s"'" % (self.pungi_user, self.mock_conf_name, _cmd1)
run_command(cmd1)
self.mock_copy(os.path.join(self.new_packages, new_tar_name),
os.path.join(self.pungi_dir, 'os/Packages/'))
def create_iso(self):
"""
创建iso的所有流程
:return: iso
"""
self.create_pungi_mock_file() # 创建/etc/mock/%s.cfg文件，并写入对应的值
self.init_pungi_mock_env() # 生成mock环境，并下载相关软件和文件
self.make_local_source()
self.do_pungi() # 制作iso并返回路径
self.create_iso_package_list() # 记录iso中所需要的package
self.copy_kylin_release_to_iso_dir() # 复制kylin_release到iso文件中
self.create_iso_md5()
self.copy_iso_to_dir() # 复制iso到upload_iso_dir中
self.copy_log_file() # 复制log文件
self.check_iso_size() # 检查iso文件大小
self.upload_iso() # 上传iso
# todo 这部分猜测时ftp和http服务器共享同一个目录文件，需要在搭建一个nginx服务器。
url = 'http://10.1.110.123/release/Release/os/%s/%s.iso'
% ('/'.join([self.tag, self.cur_year, self.cur_mon, self.cur_time]),
'-'.join([self.release, self.build, self.target_arch]))
if '10' in self.release:
url = 'http://10.1.110.123/release/Release/os/%s/%s.iso'
% ('/'.join([self.tag, self.target_arch, self.cur_year, self.cur_mon, self.cur_time]),
'-'.join(['NeoKylin-Server-10-x86', self.build, self.cur_day]))
# self.write_deploy_config(url)
# pprint.pprint(url)
self.logger.info('SUCCESS! ISO download path : %s' % url)
return True
if __name__ == '__main__':
rs = ReleaseOS()
rs.create_iso()
