debrepack.py: add 2 new fields - "dl_files" and "src_files"
Literally, they are similar with the 2 existing fields "dl_path" and
"src_path", but they are totally different.
The "src_path" points the path of package's source codes, which is
copied to build directory, besides that, some packages have additional
files outside the source codes directory, which can be achieved via the
"src_files", for example:
src_path: centos/src
src_files:
- centos/files/AAA
- centos/files/BBB
- centos/files/CCC
- ...
The special case is a package doesn't have a separate folder containing
source codes, but multi-files in different places, setting the
"src_path' as null, for example:
src_path: null
The "dl_files" is similar with "src_files", that a package has
additional files from upstream, besides the major package's tar ball via
"dl_path", for example:
dl_files:
AAA.tar.gz:
topdir: BBB
url: https:/xxx/AAA.tar.gz
md5sum: 5b80ce9a5a54938a598dc65675786142
CCC.tar.gz:
topdir: DDD
url: https://xxx/CCC.tar.gz
md5sum: 51c06d340d9572bdf7c457f98a521727
EEE.tar.gz:
topdir: FFF
url: https://xxx/EEE.tar.gz
md5sum: f8602094367d857ef91eb95d5168ad94
All the tar balls are downloaded into build directory. The "topdir" is
directory name extracted from the tar ball, that is same with the second
field of each line (separator is #) in centos_tarball-dl.lst. If the
downloaded file is not a tar ball, set the "topdir" as null.
A sample of the meta_data.yaml to support "dl_files" and "src_files"
https://review.opendev.org/c/starlingx/containers/+/818870
Isolate get_topdir() and tar_cmd() from download_tarball(). get_topdir()
is to check if a tar ball has top directory. tar_cmd() gets the corresponding
tar commandline.
Test Plan: verify the packages with dl_hook, and no side effect for
other packages.
Pass: add the "dl_files" and "src_files" to meta_data.yaml of following
packages to replace the "dl_hook", and successfully building
armada
armada-helm-toolkit
helm
chartmuseum
registry-token-server
cgcs-patch
platform-util
stx-nginx-ingress-controller-helm
nfv
sm
sm-db
sm-common
sysinv-fpga-agent
sysinv-agent
python-horizon
cert-manager-helm
stx-cert-manager-helm
Pass: successfully building with other packages.
Story: 2008846
Task: 43138
Signed-off-by: Yue Tao <yue.tao@windriver.com>
Change-Id: Icdf3c8644be312218edcbe5de9d82957abf8307e
This commit is contained in:
Yue Tao
parent
073e748cc0
commit
efdcaec0ee
|
|
@ -61,6 +61,50 @@ def get_str_md5(text):
|
|||
return str(_hash)
|
||||
|
||||
|
||||
def tar_cmd(tarball_name):
|
||||
|
||||
targz = re.match(r'.*.(tar\.gz|tar\.bz2|tar\.xz|tgz)$', tarball_name)
|
||||
if targz is None:
|
||||
self.logger.error('Not supported tarball type, the supported types are: tar.gz|tar.bz2|tar.xz|tgz')
|
||||
raise ValueError(f'{tarball_name} type is not supported')
|
||||
|
||||
# Refer to untar.py of debmake python module
|
||||
if targz == 'tar.bz2':
|
||||
cmd = 'tar --bzip2 -xf %s '
|
||||
cmdx = 'tar --bzip2 -tf %s '
|
||||
cmdc = 'tar --bzip2 -cf %s %s '
|
||||
elif targz == 'tar.xz':
|
||||
cmd = 'tar --xz -xf %s '
|
||||
cmdx = 'tar --xz -tf %s '
|
||||
cmdc = 'tar --xz -cf %s %s '
|
||||
else:
|
||||
cmd = 'tar -xzf %s '
|
||||
cmdx = 'tar -tzf %s '
|
||||
cmdc = 'tar -czf %s %s '
|
||||
|
||||
return cmd, cmdx, cmdc
|
||||
|
||||
|
||||
def get_topdir(tarball_file, logger):
|
||||
|
||||
if not os.path.exists(tarball_file):
|
||||
self.logger.error('Not such file %s', tarball_file)
|
||||
raise IOError
|
||||
|
||||
tarball_name = os.path.basename(tarball_file)
|
||||
_, cmdx, _ = tar_cmd(tarball_name)
|
||||
cmdx = cmdx + '| awk -F "/" \'{print $%s}\' | sort | uniq'
|
||||
topdir = run_shell_cmd(cmdx % (tarball_file, "1"), logger)
|
||||
subdir = run_shell_cmd(cmdx % (tarball_file, "2"), logger)
|
||||
|
||||
# The tar ball has top directory
|
||||
if len(topdir.split('\n')) == 1 and subdir != '':
|
||||
return topdir.split('\n')[0]
|
||||
# Return None if no top directory
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def run_shell_cmd(cmd, logger):
|
||||
|
||||
logger.info(f'[ Run - "{cmd}" ]')
|
||||
|
|
@ -219,8 +263,11 @@ class Parser():
|
|||
return dist + "." + str(revision)
|
||||
|
||||
src_dirname = self.meta_data["src_path"]
|
||||
if src_dirname is None:
|
||||
return dist + "." + str(revision)
|
||||
|
||||
src_path = os.path.expandvars(src_dirname)
|
||||
if not os.path.isdir(src_path):
|
||||
if not os.path.isabs(src_path):
|
||||
src_path = os.path.abspath(os.path.join(self.pkginfo["pkgpath"], src_dirname))
|
||||
if not os.path.exists(src_path):
|
||||
self.logger.error("%s: No such directory", src_path)
|
||||
|
|
@ -276,7 +323,7 @@ class Parser():
|
|||
if not os.path.exists(deb_folder):
|
||||
os.mkdir(deb_folder)
|
||||
|
||||
self.logger.info("Overwrite the debian folder by %s", deb_folder)
|
||||
self.logger.info("Overwrite the debian folder by %s", metadata)
|
||||
run_shell_cmd('cp -r %s/* %s' % (metadata, deb_folder), self.logger)
|
||||
|
||||
series = os.path.join(metadata, "patches/series")
|
||||
|
|
@ -308,6 +355,26 @@ class Parser():
|
|||
|
||||
def copy_custom_files(self):
|
||||
|
||||
if "src_files" in self.meta_data:
|
||||
for src_file in self.meta_data['src_files']:
|
||||
src_path = os.path.expandvars(src_file)
|
||||
if not os.path.exists(src_path):
|
||||
src_path = os.path.join(self.pkginfo["pkgpath"], src_file)
|
||||
if not os.path.exists(src_path):
|
||||
self.logger.error("No such file %s", src_path)
|
||||
raise IOError
|
||||
run_shell_cmd('cp -rL %s %s' % (src_path, self.pkginfo["srcdir"]),
|
||||
self.logger)
|
||||
|
||||
if "dl_files" in self.meta_data:
|
||||
for dl_file in self.meta_data['dl_files']:
|
||||
dl_file = os.path.join(self.pkginfo["packdir"], dl_file)
|
||||
if not os.path.exists(dl_file):
|
||||
self.logger.error("No such file %s", dl_file)
|
||||
raise IOError
|
||||
run_shell_cmd('cp -rL %s %s' % (dl_file, self.pkginfo["srcdir"]),
|
||||
self.logger)
|
||||
|
||||
files = os.path.join(self.pkginfo["debfolder"], "files")
|
||||
if not os.path.isdir(files) or not os.path.exists(files):
|
||||
return True
|
||||
|
|
@ -315,7 +382,7 @@ class Parser():
|
|||
for root, _, files in os.walk(files):
|
||||
for name in files:
|
||||
os.path.join(root, name)
|
||||
run_shell_cmd('cp -r %s %s' % (os.path.join(root, name), self.pkginfo["srcdir"]), self.logger)
|
||||
run_shell_cmd('cp -rL %s %s' % (os.path.join(root, name), self.pkginfo["srcdir"]), self.logger)
|
||||
|
||||
return True
|
||||
|
||||
|
|
@ -386,6 +453,36 @@ class Parser():
|
|||
|
||||
return True
|
||||
|
||||
def download_files(self):
|
||||
|
||||
if "dl_files" not in self.meta_data:
|
||||
return
|
||||
|
||||
pwd = os.getcwd()
|
||||
os.chdir(self.pkginfo["packdir"])
|
||||
for dl_file in self.meta_data['dl_files']:
|
||||
url = self.meta_data['dl_files'][dl_file]['url']
|
||||
dir_name = self.meta_data['dl_files'][dl_file]['topdir']
|
||||
download(url, dl_file, self.logger)
|
||||
if dir_name is None:
|
||||
continue
|
||||
|
||||
cmd, _, cmdc = tar_cmd(dl_file)
|
||||
# The tar ball has top directory
|
||||
if get_topdir(dl_file, self.logger) is not None:
|
||||
# Remove the top diretory
|
||||
cmd += '--strip-components 1 -C %s'
|
||||
# The tar ball is extracted under $PWD by default
|
||||
else:
|
||||
cmd += '-C %s'
|
||||
|
||||
run_shell_cmd('rm -rf %s;mkdir %s' % (dir_name, dir_name), self.logger)
|
||||
run_shell_cmd(cmd % (dl_file, dir_name), self.logger)
|
||||
run_shell_cmd(cmdc % (dl_file, dir_name), self.logger)
|
||||
run_shell_cmd('rm -rf %s' % dir_name, self.logger)
|
||||
|
||||
os.chdir(pwd)
|
||||
|
||||
def download_tarball(self):
|
||||
|
||||
tarball_name = self.meta_data["dl_path"]["name"]
|
||||
|
|
@ -406,32 +503,9 @@ class Parser():
|
|||
self.logger.error("The md5sum of %s is %s, but %s is expected", tarball_file, md5sum, tarball_md5sum)
|
||||
raise ValueError(f"The md5sum of {tarball_file} is {md5sum}, but {tarball_md5sum} is expected")
|
||||
|
||||
targz = re.match(r'.*.(tar\.gz|tar\.bz2|tar\.xz|tgz)$', tarball_name)
|
||||
if targz is None:
|
||||
self.logger.error('Not supported tarball type, the supported types are: tar.gz|tar.bz2|tar.xz|tgz')
|
||||
raise ValueError('Not supported tarball type, the supported types are: tar.gz|tar.bz2|tar.xz|tgz')
|
||||
|
||||
targz = targz.group(1)
|
||||
if targz == "tgz":
|
||||
targz = "tar.gz"
|
||||
|
||||
# Refer to untar.py of debmake python module
|
||||
if targz == 'tar.bz2':
|
||||
cmd = 'tar --bzip2 -xvf %s '
|
||||
cmdx = 'tar --bzip2 -tf %s '
|
||||
elif targz == 'tar.xz':
|
||||
cmd = 'tar --xz -xvf %s '
|
||||
cmdx = 'tar --xz -tf %s '
|
||||
else:
|
||||
cmd = 'tar -xvzf %s '
|
||||
cmdx = 'tar -tzf %s '
|
||||
|
||||
cmdx = cmdx + '| awk -F "/" \'{print $%s}\' | sort | uniq'
|
||||
topdir = run_shell_cmd(cmdx % (tarball_file, "1"), self.logger)
|
||||
subdir = run_shell_cmd(cmdx % (tarball_file, "2"), self.logger)
|
||||
|
||||
cmd, _, _ = untar_cmd(tarball_name)
|
||||
# The tar ball has top directory
|
||||
if len(topdir.split('\n')) == 1 and subdir != '':
|
||||
if get_topdir(tarball_file, self.logger) is not None:
|
||||
# Remove the top diretory
|
||||
cmd += '--strip-components 1 -C %s'
|
||||
# The tar ball is extracted under $PWD by default
|
||||
|
|
@ -440,6 +514,7 @@ class Parser():
|
|||
|
||||
os.mkdir(self.pkginfo["srcdir"])
|
||||
run_shell_cmd(cmd % (tarball_file, self.pkginfo["srcdir"]), self.logger)
|
||||
self.copy_custom_files()
|
||||
self.create_orig_tarball()
|
||||
self.update_deb_folder()
|
||||
self.apply_deb_patches()
|
||||
|
|
@ -532,15 +607,20 @@ class Parser():
|
|||
def create_src_package(self):
|
||||
|
||||
src_dirname = self.meta_data["src_path"]
|
||||
src_path = os.path.expandvars(src_dirname)
|
||||
if not os.path.isdir(src_path):
|
||||
src_path = os.path.abspath(os.path.join(self.pkginfo["pkgpath"], src_dirname))
|
||||
if not os.path.exists(src_path):
|
||||
self.logger.error("%s: No such directory", src_path)
|
||||
raise ValueError(f"{src_path}: No such directory")
|
||||
if src_dirname is None:
|
||||
os.mkdir(self.pkginfo["srcdir"])
|
||||
else:
|
||||
src_path = os.path.expandvars(src_dirname)
|
||||
if not os.path.isabs(src_path):
|
||||
src_path = os.path.abspath(os.path.join(self.pkginfo["pkgpath"], src_dirname))
|
||||
if not os.path.exists(src_path):
|
||||
self.logger.error("%s: No such directory", src_path)
|
||||
raise ValueError(f"{src_path}: No such directory")
|
||||
|
||||
# cp the .git folder, the git meta files in .git are symbol link, so need -L
|
||||
run_shell_cmd('cp -rL %s %s' % (src_path, self.pkginfo["srcdir"]), self.logger)
|
||||
# cp the .git folder, the git meta files in .git are symbol link, so need -L
|
||||
run_shell_cmd('cp -rL %s %s' % (src_path, self.pkginfo["srcdir"]), self.logger)
|
||||
|
||||
self.copy_custom_files()
|
||||
self.create_orig_tarball()
|
||||
self.update_deb_folder()
|
||||
|
||||
|
|
@ -574,6 +654,7 @@ class Parser():
|
|||
def package(self, pkgpath):
|
||||
|
||||
self.setup(pkgpath)
|
||||
self.download_files()
|
||||
if "dl_hook" in self.meta_data:
|
||||
self.run_dl_hook()
|
||||
elif "dl_path" in self.meta_data:
|
||||
|
|
@ -586,7 +667,6 @@ class Parser():
|
|||
self.download_deb_package()
|
||||
|
||||
self.apply_src_patches()
|
||||
self.copy_custom_files()
|
||||
|
||||
self.logger.info("Repackge the package %s", self.pkginfo["srcdir"])
|
||||
|
||||
|
|
@ -613,7 +693,6 @@ class Parser():
|
|||
for f in files:
|
||||
target = os.path.join(self.output, f)
|
||||
source = os.path.join(self.pkginfo["packdir"], f)
|
||||
self.logger.info("Copy %s to %s", source, target)
|
||||
shutil.copy(source, self.output)
|
||||
run_shell_cmd('cp -Lr %s %s' % (source, self.output), self.logger)
|
||||
|
||||
return files
|
||||
|
|
|
|||
Loading…
Reference in New Issue