efdcaec0ee
Literally, they are similar with the 2 existing fields "dl_path" and
"src_path", but they are totally different.
The "src_path" points the path of package's source codes, which is
copied to build directory, besides that, some packages have additional
files outside the source codes directory, which can be achieved via the
"src_files", for example:
src_path: centos/src
src_files:
- centos/files/AAA
- centos/files/BBB
- centos/files/CCC
- ...
The special case is a package doesn't have a separate folder containing
source codes, but multi-files in different places, setting the
"src_path' as null, for example:
src_path: null
The "dl_files" is similar with "src_files", that a package has
additional files from upstream, besides the major package's tar ball via
"dl_path", for example:
dl_files:
AAA.tar.gz:
topdir: BBB
url: https:/xxx/AAA.tar.gz
md5sum: 5b80ce9a5a54938a598dc65675786142
CCC.tar.gz:
topdir: DDD
url: https://xxx/CCC.tar.gz
md5sum: 51c06d340d9572bdf7c457f98a521727
EEE.tar.gz:
topdir: FFF
url: https://xxx/EEE.tar.gz
md5sum: f8602094367d857ef91eb95d5168ad94
All the tar balls are downloaded into build directory. The "topdir" is
directory name extracted from the tar ball, that is same with the second
field of each line (separator is #) in centos_tarball-dl.lst. If the
downloaded file is not a tar ball, set the "topdir" as null.
A sample of the meta_data.yaml to support "dl_files" and "src_files"
https://review.opendev.org/c/starlingx/containers/+/818870
Isolate get_topdir() and tar_cmd() from download_tarball(). get_topdir()
is to check if a tar ball has top directory. tar_cmd() gets the corresponding
tar commandline.
Test Plan: verify the packages with dl_hook, and no side effect for
other packages.
Pass: add the "dl_files" and "src_files" to meta_data.yaml of following
packages to replace the "dl_hook", and successfully building
armada
armada-helm-toolkit
helm
chartmuseum
registry-token-server
cgcs-patch
platform-util
stx-nginx-ingress-controller-helm
nfv
sm
sm-db
sm-common
sysinv-fpga-agent
sysinv-agent
python-horizon
cert-manager-helm
stx-cert-manager-helm
Pass: successfully building with other packages.
Story: 2008846
Task: 43138
Signed-off-by: Yue Tao <yue.tao@windriver.com>
Change-Id: Icdf3c8644be312218edcbe5de9d82957abf8307e
699 lines
26 KiB
Python
Executable File
699 lines
26 KiB
Python
Executable File
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
# Copyright (C) 2021 WindRiver Corporation
|
|
|
|
# import apt
|
|
import apt_pkg
|
|
import debian.deb822
|
|
from debian.debian_support import BaseVersion
|
|
import git
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import progressbar
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
# import urllib.request
|
|
import yaml
|
|
|
|
|
|
RELEASENOTES = " ".join([os.environ.get('PROJECT'), os.environ.get('MY_RELEASE'), "distribution"])
|
|
DIST = os.environ.get('STX_DIST')
|
|
|
|
|
|
class DownloadProgress():
|
|
def __init__(self):
|
|
self.pbar = None
|
|
|
|
def __call__(self, block_num, block_size, total_size):
|
|
|
|
if total_size < 0:
|
|
return
|
|
|
|
if not self.pbar:
|
|
self.pbar = progressbar.ProgressBar(maxval=total_size)
|
|
self.pbar.start()
|
|
|
|
downloaded = block_num * block_size
|
|
if downloaded < total_size:
|
|
self.pbar.update(downloaded)
|
|
else:
|
|
self.pbar.finish()
|
|
|
|
|
|
def get_str_md5(text):
|
|
|
|
md5obj = hashlib.md5()
|
|
md5obj.update(text.encode())
|
|
_hash = md5obj.hexdigest()
|
|
return str(_hash)
|
|
|
|
|
|
def tar_cmd(tarball_name):
|
|
|
|
targz = re.match(r'.*.(tar\.gz|tar\.bz2|tar\.xz|tgz)$', tarball_name)
|
|
if targz is None:
|
|
self.logger.error('Not supported tarball type, the supported types are: tar.gz|tar.bz2|tar.xz|tgz')
|
|
raise ValueError(f'{tarball_name} type is not supported')
|
|
|
|
# Refer to untar.py of debmake python module
|
|
if targz == 'tar.bz2':
|
|
cmd = 'tar --bzip2 -xf %s '
|
|
cmdx = 'tar --bzip2 -tf %s '
|
|
cmdc = 'tar --bzip2 -cf %s %s '
|
|
elif targz == 'tar.xz':
|
|
cmd = 'tar --xz -xf %s '
|
|
cmdx = 'tar --xz -tf %s '
|
|
cmdc = 'tar --xz -cf %s %s '
|
|
else:
|
|
cmd = 'tar -xzf %s '
|
|
cmdx = 'tar -tzf %s '
|
|
cmdc = 'tar -czf %s %s '
|
|
|
|
return cmd, cmdx, cmdc
|
|
|
|
|
|
def get_topdir(tarball_file, logger):
|
|
|
|
if not os.path.exists(tarball_file):
|
|
self.logger.error('Not such file %s', tarball_file)
|
|
raise IOError
|
|
|
|
tarball_name = os.path.basename(tarball_file)
|
|
_, cmdx, _ = tar_cmd(tarball_name)
|
|
cmdx = cmdx + '| awk -F "/" \'{print $%s}\' | sort | uniq'
|
|
topdir = run_shell_cmd(cmdx % (tarball_file, "1"), logger)
|
|
subdir = run_shell_cmd(cmdx % (tarball_file, "2"), logger)
|
|
|
|
# The tar ball has top directory
|
|
if len(topdir.split('\n')) == 1 and subdir != '':
|
|
return topdir.split('\n')[0]
|
|
# Return None if no top directory
|
|
else:
|
|
return None
|
|
|
|
|
|
def run_shell_cmd(cmd, logger):
|
|
|
|
logger.info(f'[ Run - "{cmd}" ]')
|
|
try:
|
|
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
universal_newlines=True, shell=True)
|
|
# process.wait()
|
|
outs, errs = process.communicate()
|
|
except Exception:
|
|
process.kill()
|
|
outs, errs = process.communicate()
|
|
logger.error(f'[ Failed - "{cmd}" ]')
|
|
raise Exception(f'[ Failed - "{cmd}" ]')
|
|
|
|
for log in outs.strip().split("\n"):
|
|
if log != "":
|
|
logger.debug(log.strip())
|
|
|
|
if process.returncode != 0:
|
|
for log in errs.strip().split("\n"):
|
|
logger.error(log)
|
|
logger.error(f'[ Failed - "{cmd}" ]')
|
|
raise Exception(f'[ Failed - "{cmd}" ]')
|
|
|
|
return outs.strip()
|
|
|
|
|
|
def download(url, savepath, logger):
|
|
|
|
logger.info(f"Download {url} to {savepath}")
|
|
download_cmd = "wget -t 5 --wait=15 %s -O %s"
|
|
run_shell_cmd(download_cmd % (url, savepath), logger)
|
|
# urllib.request.urlretrieve(url, savepath, DownloadProgress())
|
|
|
|
return True
|
|
|
|
|
|
def is_git_repo(path):
|
|
try:
|
|
_ = git.Repo(path).git_dir
|
|
return True
|
|
except git.exc.InvalidGitRepositoryError:
|
|
return False
|
|
|
|
|
|
class Parser():
|
|
level_relations = {
|
|
'debug': logging.DEBUG,
|
|
'info': logging.INFO,
|
|
'warning': logging.WARNING,
|
|
'error': logging.ERROR,
|
|
'crit': logging.CRITICAL
|
|
}
|
|
|
|
def __init__(self, basedir, output, loglevel='info', srcrepo=None):
|
|
|
|
self.logger = logging.getLogger(__name__)
|
|
self.logger.setLevel(self.level_relations.get(loglevel))
|
|
formatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
|
|
console = logging.StreamHandler(sys.stdout)
|
|
console.setFormatter(formatter)
|
|
self.logger.addHandler(console)
|
|
|
|
if not os.path.isdir(basedir):
|
|
self.logger.error("%s: No such file or directory", basedir)
|
|
raise Exception(f"{basedir}: No such file or directory")
|
|
self.basedir = os.path.abspath(basedir)
|
|
|
|
if not os.path.isdir(output):
|
|
self.logger.error("%s: No such file or directory", output)
|
|
raise Exception(f"{output}: No such file or directory")
|
|
self.output = os.path.abspath(output)
|
|
|
|
self.srcrepo = srcrepo
|
|
self.meta_data = dict()
|
|
self.versions = dict()
|
|
self.pkginfo = dict()
|
|
|
|
def setup(self, pkgpath):
|
|
|
|
if not os.path.isdir(pkgpath):
|
|
self.logger.error("%s: No such file or directory", pkgpath)
|
|
raise Exception(f"{pkgpath}: No such file or directory")
|
|
|
|
self.pkginfo["pkgpath"] = os.path.abspath(pkgpath)
|
|
self.pkginfo["pkgname"] = os.path.basename(pkgpath)
|
|
self.pkginfo["packdir"] = os.path.join(self.basedir, self.pkginfo["pkgname"])
|
|
if not os.path.exists(self.pkginfo["packdir"]):
|
|
os.mkdir(self.pkginfo["packdir"])
|
|
|
|
logfile = os.path.join(self.pkginfo["packdir"], self.pkginfo["pkgname"] + ".log")
|
|
if os.path.exists(logfile):
|
|
os.remove(logfile)
|
|
logfile_handler = logging.FileHandler(logfile, 'w')
|
|
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
|
logfile_handler.setFormatter(formatter)
|
|
self.logger.addHandler(logfile_handler)
|
|
|
|
self.pkginfo["debfolder"] = os.path.join(self.pkginfo["pkgpath"], "debian")
|
|
if not os.path.isdir(self.pkginfo["debfolder"]):
|
|
self.logger.error("No debian folder")
|
|
raise Exception("No debian folder")
|
|
|
|
meta_data = os.path.join(self.pkginfo["debfolder"], "meta_data.yaml")
|
|
if not os.path.exists(meta_data):
|
|
self.logger.error("Not find meta_data.yaml")
|
|
raise Exception("Not find meta_data.yaml")
|
|
with open(meta_data) as f:
|
|
self.meta_data = yaml.full_load(f)
|
|
|
|
if "debver" not in self.meta_data:
|
|
self.logger.error("No debver defined in meta_data.yaml")
|
|
raise Exception("No debver defined in meta_data.yaml")
|
|
|
|
if "debname" in self.meta_data:
|
|
self.pkginfo["pkgname"] = self.meta_data["debname"]
|
|
|
|
self.versions["full_version"] = str(self.meta_data["debver"])
|
|
self.versions["upstream_version"] = BaseVersion(self.versions["full_version"]).upstream_version
|
|
self.versions["debian_revision"] = BaseVersion(self.versions["full_version"]).debian_revision
|
|
self.versions["epoch"] = BaseVersion(self.versions["full_version"]).epoch
|
|
|
|
self.logger.info("=== Package Name: %s", self.pkginfo["pkgname"])
|
|
self.logger.info("=== Package Version: %s", self.versions["full_version"])
|
|
self.logger.info("=== Package Path: %s", self.pkginfo["pkgpath"])
|
|
|
|
srcdir = self.pkginfo["pkgname"] + "-" + self.versions["upstream_version"]
|
|
self.pkginfo["srcdir"] = os.path.join(self.pkginfo["packdir"], srcdir)
|
|
if os.path.exists(self.pkginfo["srcdir"]):
|
|
shutil.rmtree(self.pkginfo["srcdir"])
|
|
|
|
def set_revision(self):
|
|
|
|
revision = 0
|
|
dist = ""
|
|
if "revision" not in self.meta_data:
|
|
return dist
|
|
|
|
revision_data = self.meta_data["revision"]
|
|
if "dist" in revision_data:
|
|
if revision_data["dist"] is not None:
|
|
dist = os.path.expandvars(revision_data["dist"])
|
|
|
|
git_rev_list = "cd %s;git rev-list --count HEAD ."
|
|
git_rev_list_from = "cd %s;git rev-list --count %s..HEAD ."
|
|
git_status = "cd %s;git status --porcelain . | wc -l"
|
|
|
|
if "PKG_GITREVCOUNT" in revision_data:
|
|
if "PKG_BASE_SRCREV" in revision_data:
|
|
revision = int(run_shell_cmd(git_rev_list_from % (self.pkginfo["debfolder"], revision_data["PKG_BASE_SRCREV"]), self.logger))
|
|
else:
|
|
revision = int(run_shell_cmd(git_rev_list % self.pkginfo["debfolder"], self.logger))
|
|
revision += int(run_shell_cmd(git_status % self.pkginfo["debfolder"], self.logger))
|
|
|
|
if "src_path" not in self.meta_data:
|
|
return dist + "." + str(revision)
|
|
|
|
src_dirname = self.meta_data["src_path"]
|
|
if src_dirname is None:
|
|
return dist + "." + str(revision)
|
|
|
|
src_path = os.path.expandvars(src_dirname)
|
|
if not os.path.isabs(src_path):
|
|
src_path = os.path.abspath(os.path.join(self.pkginfo["pkgpath"], src_dirname))
|
|
if not os.path.exists(src_path):
|
|
self.logger.error("%s: No such directory", src_path)
|
|
raise ValueError(f"{src_path}: No such directory")
|
|
|
|
if "SRC_GITREVCOUNT" in revision_data:
|
|
if "SRC_BASE_SRCREV" not in revision_data:
|
|
self.logger.error("SRC_BASE_SRCREV must be set")
|
|
raise ValueError("SRC_BASE_SRCREV must be set")
|
|
revision += int(run_shell_cmd(git_rev_list_from % (src_path, revision_data["SRC_BASE_SRCREV"]), self.logger))
|
|
revision += int(run_shell_cmd(git_status % src_path, self.logger))
|
|
|
|
return dist + "." + str(revision)
|
|
|
|
def checksum(self, pkgpath):
|
|
|
|
if not os.path.isdir(pkgpath):
|
|
self.logger.error("%s: No such file or directory", pkgpath)
|
|
raise Exception(f"{pkgpath}: No such file or directory")
|
|
|
|
debfolder = os.path.join(pkgpath, "debian")
|
|
if not os.path.isdir(debfolder):
|
|
self.logger.error("%s: no such directory", debfolder)
|
|
raise Exception(f"{debfolder}: no such directory")
|
|
|
|
content = ""
|
|
for root, _, files in os.walk(debfolder):
|
|
for name in files:
|
|
f = open(os.path.join(root, name), 'r', encoding="ISO-8859-1")
|
|
content += f.read()
|
|
|
|
return get_str_md5(content)
|
|
|
|
def set_deb_format(self):
|
|
|
|
deb_format = run_shell_cmd('dpkg-source --print-format %s' % self.pkginfo["srcdir"], self.logger)
|
|
if re.match("1.0", deb_format):
|
|
return "1.0", None
|
|
|
|
format_ver, format_type = deb_format.split(" ")
|
|
format_ver = format_ver.strip()
|
|
format_type = format_type.strip("()")
|
|
|
|
return format_ver, format_type
|
|
|
|
def update_deb_folder(self):
|
|
|
|
metadata = os.path.join(self.pkginfo["debfolder"], "deb_folder")
|
|
if not os.path.isdir(metadata):
|
|
return True
|
|
|
|
deb_folder = os.path.join(self.pkginfo["srcdir"], "debian")
|
|
if not os.path.exists(deb_folder):
|
|
os.mkdir(deb_folder)
|
|
|
|
self.logger.info("Overwrite the debian folder by %s", metadata)
|
|
run_shell_cmd('cp -r %s/* %s' % (metadata, deb_folder), self.logger)
|
|
|
|
series = os.path.join(metadata, "patches/series")
|
|
if not os.path.isfile(series):
|
|
return True
|
|
|
|
format_ver, format_type = self.set_deb_format()
|
|
if format_type == "quilt" and format_ver == "3.0":
|
|
return True
|
|
|
|
f = open(series)
|
|
patches = f.readlines()
|
|
patches_src = os.path.dirname(series)
|
|
f.close()
|
|
|
|
pwd = os.getcwd()
|
|
os.chdir(self.pkginfo["srcdir"])
|
|
for patch in patches:
|
|
patch_file = patch.strip()
|
|
# Skip comment lines and blank lines
|
|
if patch_file.startswith('#') or patch_file == "":
|
|
continue
|
|
self.logger.info("Apply src patch: %s", patch_file)
|
|
patch = os.path.join(patches_src, patch_file)
|
|
run_shell_cmd('patch -p1 < %s' % patch, self.logger)
|
|
os.chdir(pwd)
|
|
|
|
return True
|
|
|
|
def copy_custom_files(self):
|
|
|
|
if "src_files" in self.meta_data:
|
|
for src_file in self.meta_data['src_files']:
|
|
src_path = os.path.expandvars(src_file)
|
|
if not os.path.exists(src_path):
|
|
src_path = os.path.join(self.pkginfo["pkgpath"], src_file)
|
|
if not os.path.exists(src_path):
|
|
self.logger.error("No such file %s", src_path)
|
|
raise IOError
|
|
run_shell_cmd('cp -rL %s %s' % (src_path, self.pkginfo["srcdir"]),
|
|
self.logger)
|
|
|
|
if "dl_files" in self.meta_data:
|
|
for dl_file in self.meta_data['dl_files']:
|
|
dl_file = os.path.join(self.pkginfo["packdir"], dl_file)
|
|
if not os.path.exists(dl_file):
|
|
self.logger.error("No such file %s", dl_file)
|
|
raise IOError
|
|
run_shell_cmd('cp -rL %s %s' % (dl_file, self.pkginfo["srcdir"]),
|
|
self.logger)
|
|
|
|
files = os.path.join(self.pkginfo["debfolder"], "files")
|
|
if not os.path.isdir(files) or not os.path.exists(files):
|
|
return True
|
|
|
|
for root, _, files in os.walk(files):
|
|
for name in files:
|
|
os.path.join(root, name)
|
|
run_shell_cmd('cp -rL %s %s' % (os.path.join(root, name), self.pkginfo["srcdir"]), self.logger)
|
|
|
|
return True
|
|
|
|
def apply_src_patches(self):
|
|
|
|
format_ver, format_type = self.set_deb_format()
|
|
series = os.path.join(self.pkginfo["debfolder"], "patches/series")
|
|
if not os.path.isfile(series):
|
|
return True
|
|
|
|
f = open(series)
|
|
patches = f.readlines()
|
|
patches_src = os.path.dirname(series)
|
|
f.close()
|
|
|
|
patches_folder = os.path.join(self.pkginfo["srcdir"], "debian/patches")
|
|
series_file = os.path.join(self.pkginfo["srcdir"], "debian/patches/series")
|
|
if not os.path.isdir(patches_folder):
|
|
os.mkdir(patches_folder)
|
|
os.mknod(series_file)
|
|
|
|
pwd = os.getcwd()
|
|
os.chdir(self.pkginfo["srcdir"])
|
|
for patch in patches:
|
|
patch_file = patch.strip()
|
|
# Skip comment lines and blank lines
|
|
if patch_file.startswith('#') or patch_file == "":
|
|
continue
|
|
self.logger.info("Apply src patch: %s", patch_file)
|
|
patch = os.path.join(patches_src, patch_file)
|
|
if format_ver == "1.0":
|
|
run_shell_cmd('patch -p1 < %s' % patch, self.logger)
|
|
else:
|
|
if format_type == "quilt":
|
|
run_shell_cmd('cp -r %s %s' % (patch, patches_folder), self.logger)
|
|
with open(series_file, 'a') as f:
|
|
f.write(patch_file + "\n")
|
|
f.close()
|
|
elif format_type == "native":
|
|
run_shell_cmd('patch -p1 < %s' % patch, self.logger)
|
|
else:
|
|
self.logger.error('Invalid deb format: %s %s', format_ver, format_type)
|
|
raise Exception(f'[ Invalid deb format: {format_ver} {format_type} ]')
|
|
|
|
os.chdir(pwd)
|
|
return True
|
|
|
|
def apply_deb_patches(self):
|
|
|
|
series = os.path.join(self.pkginfo["debfolder"], "deb_patches/series")
|
|
if not os.path.isfile(series):
|
|
return True
|
|
f = open(series)
|
|
patches = f.readlines()
|
|
patches_src = os.path.dirname(series)
|
|
|
|
pwd = os.getcwd()
|
|
os.chdir(self.pkginfo["srcdir"])
|
|
for patch in patches:
|
|
patch_file = patch.strip()
|
|
# Skip comment lines and blank lines
|
|
if patch_file.startswith('#') or patch_file == "":
|
|
continue
|
|
self.logger.info("Apply deb patch: %s", patch_file)
|
|
patch = os.path.join(patches_src, patch_file)
|
|
run_shell_cmd("patch -p1 < %s" % patch, self.logger)
|
|
os.chdir(pwd)
|
|
|
|
return True
|
|
|
|
def download_files(self):
|
|
|
|
if "dl_files" not in self.meta_data:
|
|
return
|
|
|
|
pwd = os.getcwd()
|
|
os.chdir(self.pkginfo["packdir"])
|
|
for dl_file in self.meta_data['dl_files']:
|
|
url = self.meta_data['dl_files'][dl_file]['url']
|
|
dir_name = self.meta_data['dl_files'][dl_file]['topdir']
|
|
download(url, dl_file, self.logger)
|
|
if dir_name is None:
|
|
continue
|
|
|
|
cmd, _, cmdc = tar_cmd(dl_file)
|
|
# The tar ball has top directory
|
|
if get_topdir(dl_file, self.logger) is not None:
|
|
# Remove the top diretory
|
|
cmd += '--strip-components 1 -C %s'
|
|
# The tar ball is extracted under $PWD by default
|
|
else:
|
|
cmd += '-C %s'
|
|
|
|
run_shell_cmd('rm -rf %s;mkdir %s' % (dir_name, dir_name), self.logger)
|
|
run_shell_cmd(cmd % (dl_file, dir_name), self.logger)
|
|
run_shell_cmd(cmdc % (dl_file, dir_name), self.logger)
|
|
run_shell_cmd('rm -rf %s' % dir_name, self.logger)
|
|
|
|
os.chdir(pwd)
|
|
|
|
def download_tarball(self):
|
|
|
|
tarball_name = self.meta_data["dl_path"]["name"]
|
|
tarball_url = self.meta_data["dl_path"]["url"]
|
|
tarball_md5sum = self.meta_data["dl_path"]["md5sum"]
|
|
|
|
tarball_file = os.path.join(self.pkginfo["packdir"], tarball_name)
|
|
if os.path.exists(tarball_file):
|
|
md5sum = run_shell_cmd('md5sum %s |cut -d" " -f1' % tarball_file, self.logger)
|
|
if md5sum != tarball_md5sum:
|
|
self.logger.info("The md5sum of existing %s is %s, but %s is expected, redownload", tarball_file, md5sum, tarball_md5sum)
|
|
os.remove(tarball_file)
|
|
|
|
if not os.path.exists(tarball_file):
|
|
download(tarball_url, tarball_file, self.logger)
|
|
md5sum = run_shell_cmd('md5sum %s |cut -d" " -f1' % tarball_file, self.logger)
|
|
if md5sum != tarball_md5sum:
|
|
self.logger.error("The md5sum of %s is %s, but %s is expected", tarball_file, md5sum, tarball_md5sum)
|
|
raise ValueError(f"The md5sum of {tarball_file} is {md5sum}, but {tarball_md5sum} is expected")
|
|
|
|
cmd, _, _ = untar_cmd(tarball_name)
|
|
# The tar ball has top directory
|
|
if get_topdir(tarball_file, self.logger) is not None:
|
|
# Remove the top diretory
|
|
cmd += '--strip-components 1 -C %s'
|
|
# The tar ball is extracted under $PWD by default
|
|
else:
|
|
cmd += '-C %s'
|
|
|
|
os.mkdir(self.pkginfo["srcdir"])
|
|
run_shell_cmd(cmd % (tarball_file, self.pkginfo["srcdir"]), self.logger)
|
|
self.copy_custom_files()
|
|
self.create_orig_tarball()
|
|
self.update_deb_folder()
|
|
self.apply_deb_patches()
|
|
|
|
return True
|
|
|
|
def upload_deb_package(self):
|
|
|
|
self.logger.info("Uploading the dsc files of %s to local repo %s", self.pkginfo["pkgname"], self.srcrepo)
|
|
# strip epoch
|
|
ver = self.versions["full_version"].split(":")[-1]
|
|
dsc_file = os.path.join(self.pkginfo["packdir"], self.pkginfo["pkgname"] + "_" + ver + ".dsc")
|
|
|
|
cmd = "repo_manage.py upload_pkg -p %s -r %s"
|
|
run_shell_cmd(cmd % (dsc_file, self.srcrepo), self.logger)
|
|
|
|
return True
|
|
|
|
def download_deb_archive(self):
|
|
|
|
archive_url = self.meta_data["archive"]
|
|
dsc_filename = self.pkginfo["pkgname"] + "_" + self.versions["full_version"] + ".dsc"
|
|
dsc_file = os.path.join(archive_url, dsc_filename)
|
|
local_dsc = os.path.join(self.pkginfo["packdir"], dsc_filename)
|
|
download(dsc_file, local_dsc, self.logger)
|
|
with open(local_dsc) as f:
|
|
c = debian.deb822.Dsc(f)
|
|
|
|
for f in c['Files']:
|
|
local_f = os.path.join(self.pkginfo["packdir"], f['name'])
|
|
remote_f = os.path.join(archive_url, f['name'])
|
|
download(remote_f, local_f, self.logger)
|
|
run_shell_cmd("cd %s;dpkg-source -x %s" % (self.pkginfo["packdir"], dsc_filename), self.logger)
|
|
|
|
self.apply_deb_patches()
|
|
|
|
return True
|
|
|
|
def download_deb_package(self):
|
|
|
|
fullname = self.pkginfo["pkgname"] + "=" + self.versions["full_version"]
|
|
supported_versions = list()
|
|
|
|
apt_pkg.init()
|
|
sources = apt_pkg.SourceRecords()
|
|
source_lookup = sources.lookup(self.pkginfo["pkgname"])
|
|
while source_lookup and self.versions["full_version"] != sources.version:
|
|
supported_versions.append(sources.version)
|
|
source_lookup = sources.lookup(self.pkginfo["pkgname"])
|
|
|
|
if not source_lookup:
|
|
self.logger.error("No source for %s", fullname)
|
|
self.logger.info("The supported versions are %s", supported_versions)
|
|
raise ValueError(f"No source for {fullname}")
|
|
self.logger.info("Found %s", fullname)
|
|
|
|
self.logger.info("Fetch %s to %s", fullname, self.pkginfo["packdir"])
|
|
# first_binary = sources.binaries[0]
|
|
# cache = apt.cache.Cache()
|
|
# package = cache[first_binary]
|
|
# package.candidate.fetch_source(destdir=self.pkginfo["packdir"],unpack=True)
|
|
run_shell_cmd("cd %s; apt-get source %s" % (self.pkginfo["packdir"], fullname), self.logger)
|
|
|
|
self.logger.info("Deploy %s to %s", fullname, self.pkginfo["srcdir"])
|
|
if self.srcrepo is not None:
|
|
self.upload_deb_package()
|
|
self.apply_deb_patches()
|
|
|
|
return True
|
|
|
|
def create_orig_tarball(self):
|
|
|
|
if not os.path.exists(self.pkginfo["srcdir"]):
|
|
self.logger.error("%s: no such directory", self.pkginfo["srcdir"])
|
|
raise ValueError(f'{self.pkginfo["srcdir"]}: no such directory')
|
|
|
|
if is_git_repo(self.pkginfo["srcdir"]):
|
|
debian_folder = os.path.join(self.pkginfo["srcdir"], "debian")
|
|
if os.path.exists(debian_folder):
|
|
self.logger.info("Generate orig tarballs from git repositry %s", self.pkginfo["srcdir"])
|
|
run_shell_cmd('cd %s; gbp export-orig --upstream-tree=HEAD' % self.pkginfo["srcdir"], self.logger)
|
|
return
|
|
# remove .git directory
|
|
run_shell_cmd('rm -rf %s' % os.path.join(self.pkginfo["srcdir"], ".git"), self.logger)
|
|
|
|
srcname = os.path.basename(self.pkginfo["srcdir"])
|
|
origtargz = self.pkginfo["pkgname"] + '_' + self.versions["upstream_version"] + '.orig.tar.gz'
|
|
run_shell_cmd('cd %s; tar czvf %s %s' % (self.pkginfo["packdir"], origtargz, srcname), self.logger)
|
|
|
|
def create_src_package(self):
|
|
|
|
src_dirname = self.meta_data["src_path"]
|
|
if src_dirname is None:
|
|
os.mkdir(self.pkginfo["srcdir"])
|
|
else:
|
|
src_path = os.path.expandvars(src_dirname)
|
|
if not os.path.isabs(src_path):
|
|
src_path = os.path.abspath(os.path.join(self.pkginfo["pkgpath"], src_dirname))
|
|
if not os.path.exists(src_path):
|
|
self.logger.error("%s: No such directory", src_path)
|
|
raise ValueError(f"{src_path}: No such directory")
|
|
|
|
# cp the .git folder, the git meta files in .git are symbol link, so need -L
|
|
run_shell_cmd('cp -rL %s %s' % (src_path, self.pkginfo["srcdir"]), self.logger)
|
|
|
|
self.copy_custom_files()
|
|
self.create_orig_tarball()
|
|
self.update_deb_folder()
|
|
|
|
return True
|
|
|
|
def run_dl_hook(self):
|
|
|
|
dl_hook = self.meta_data["dl_hook"]
|
|
if not os.path.isabs(dl_hook):
|
|
dl_hook = os.path.join(self.pkginfo["debfolder"], dl_hook)
|
|
if not os.path.exists(dl_hook):
|
|
self.logger.error("%s doesn't exist", dl_hook)
|
|
raise ValueError(f"{dl_hook} doesn't exist")
|
|
run_shell_cmd('cp -r %s %s' % (dl_hook, self.pkginfo["packdir"]), self.logger)
|
|
|
|
pwd = os.getcwd()
|
|
os.chdir(self.pkginfo["packdir"])
|
|
if not os.access("dl_hook", os.X_OK):
|
|
self.logger.error("dl_hook can't execute")
|
|
raise ValueError("dl_hook can't execute")
|
|
run_shell_cmd('./dl_hook %s' % os.path.basename(self.pkginfo["srcdir"]), self.logger)
|
|
origtar = self.pkginfo["pkgname"] + '_' + self.versions["upstream_version"]
|
|
origtargz = origtar + '.orig.tar.gz'
|
|
origtarxz = origtar + '.orig.tar.xz'
|
|
if not os.path.exists(origtargz) and not os.path.exists(origtarxz):
|
|
self.create_orig_tarball()
|
|
os.chdir(pwd)
|
|
self.update_deb_folder()
|
|
self.apply_deb_patches()
|
|
|
|
def package(self, pkgpath):
|
|
|
|
self.setup(pkgpath)
|
|
self.download_files()
|
|
if "dl_hook" in self.meta_data:
|
|
self.run_dl_hook()
|
|
elif "dl_path" in self.meta_data:
|
|
self.download_tarball()
|
|
elif "src_path" in self.meta_data:
|
|
self.create_src_package()
|
|
elif "archive" in self.meta_data:
|
|
self.download_deb_archive()
|
|
else:
|
|
self.download_deb_package()
|
|
|
|
self.apply_src_patches()
|
|
|
|
self.logger.info("Repackge the package %s", self.pkginfo["srcdir"])
|
|
|
|
changelog = os.path.join(self.pkginfo["srcdir"], 'debian/changelog')
|
|
src = run_shell_cmd('dpkg-parsechangelog -l %s --show-field source' % changelog, self.logger)
|
|
ver = run_shell_cmd('dpkg-parsechangelog -l %s --show-field version' % changelog, self.logger)
|
|
ver += self.set_revision()
|
|
run_shell_cmd('cd %s; dch -p -D stable -v %s %s' % (self.pkginfo["srcdir"], ver, RELEASENOTES), self.logger)
|
|
# strip epoch
|
|
ver = ver.split(":")[-1]
|
|
|
|
# Skip building(-S) and skip checking dependence(-d)
|
|
run_shell_cmd('cd %s; dpkg-buildpackage -nc -us -uc -S -d' % self.pkginfo["srcdir"], self.logger)
|
|
|
|
dsc_file = src + "_" + ver + ".dsc"
|
|
with open(os.path.join(self.pkginfo["packdir"], dsc_file)) as f:
|
|
c = debian.deb822.Dsc(f)
|
|
|
|
files = list()
|
|
files.append(dsc_file)
|
|
for f in c['Files']:
|
|
files.append(f['name'])
|
|
|
|
for f in files:
|
|
target = os.path.join(self.output, f)
|
|
source = os.path.join(self.pkginfo["packdir"], f)
|
|
run_shell_cmd('cp -Lr %s %s' % (source, self.output), self.logger)
|
|
|
|
return files
|