starlingx/root
Code Issues Proposed changes

Merge "build-pkgs: Fixed the deb packages missing issue after reuse"

Browse Source
This commit is contained in:
Zuul 2023-05-25 19:56:44 +00:00 committed by Gerrit Code Review
commit 6ef308a6f6
3 changed files with 168 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
build-tools/stx/build-pkgs
View File

@ -457,6 +457,56 @@ def get_package_jobs(pkg_dir, distro=STX_DEFAULT_DISTRO):
return int(jobs)
def get_never_reuse_pkgs():
never_reuse_pkgs = set()
lst_dir = os.path.join(os.environ.get('MY_REPO_ROOT_DIR'),
'stx-tools/debian-mirror-tools/config/debian/common')
never_reuse_lst = os.path.join(lst_dir, 'never_reuse.lst')
try:
with open(never_reuse_lst, 'r') as npkgs:
lines = list(line for line in (p.strip() for p in npkgs) if line)
except Exception as e:
logger.warning(str(e))
return never_reuse_pkgs
else:
for pkg in lines:
pkg = pkg.strip()
if pkg.startswith('#'):
continue
never_reuse_pkgs.add(pkg)
return never_reuse_pkgs
def move_debs_to_build_dir(dl_bin_debs_dir):
try:
for root, dirs, files in os.walk(dl_bin_debs_dir):
if dirs:
pass
for r in files:
if r.endswith('.deb'):
pkg_item = r.split('_')
sdeb = '_'.join([pkg_item[0], pkg_item[1]])
pname = ''
for btype in ['std', 'rt']:
debs_clue = get_debs_clue(btype)
deb_file = os.path.join(root, r)
pname = debsentry.get_pkg_by_deb(debs_clue, sdeb, logger)
if pname:
pkg_build_dir = os.path.join(BUILD_ROOT, btype, pname)
os.makedirs(pkg_build_dir, exist_ok=True)
os.system('sudo rm -f %s/*.build' % (pkg_build_dir))
shutil.move(deb_file, os.path.join(pkg_build_dir, r))
logger.debug("Reuse: %s is moved to build directory", sdeb)
break
if not pname:
logger.warning("Failed to get the package name for %s", sdeb)
except Exception as e:
logger.error("An exception occurred during moving reused debs into build directory")
logger.error(str(e))
return False
return True
class repoSnapshots():
"""
The repository snapshots pool to manage the apply/release
@ -504,6 +554,7 @@ class BuildController():
'upload_source': False,
'poll_build_status': True,
'reuse': False,
'reuse_max': False,
'build_all': False,
'reuse_export': True,
'dl_reused': False,
@ -650,6 +701,9 @@ class BuildController():
os.makedirs(caches_dir, exist_ok=True)
self.lists['pkgs_not_found'] = []
self.lists['never_reuse_pkgs'] = []
if self.attrs['reuse'] and not self.attrs['reuse_max']:
self.lists['never_reuse_pkgs'] = get_never_reuse_pkgs()
for build_type in build_types_to_init:
self.lists['success_' + build_type] = []
@ -736,10 +790,17 @@ class BuildController():
def download_reused_debs(self, distribution):
if not self.attrs['dl_reused']:
return True
try:
reuse_dl_dir = os.path.join(BUILD_ROOT, 'reused_debs')
if os.path.exists(reuse_dl_dir):
shutil.rmtree(reuse_dl_dir)
if os.path.exists(reuse_dl_dir):
logger.error("Failed to clean the old download directory")
logger.error("Please check and make sure it is removed")
return False
os.makedirs(reuse_dl_dir, exist_ok=True)
apt_src_file = os.path.join(BUILD_ROOT, 'aptsrc')
try:
with open(apt_src_file, 'w') as f:
reuse_url = os.environ.get('STX_SHARED_REPO')
apt_item = ' '.join(['deb [trusted=yes]', reuse_url, distribution, 'main\n'])
@ -771,9 +832,12 @@ class BuildController():
return False
if len(fetch_ret['deb-failed']) == 0:
logger.info("Successfully downloaded all reused debs to %s", reuse_dl_dir + '/downloads/binary')
dl_bin_debs_dir=os.path.join(reuse_dl_dir, 'downloads/binary')
logger.info("Successfully downloaded all reused debs to %s", dl_bin_debs_dir)
move_debs_to_build_dir(dl_bin_debs_dir)
return True
else:
for failed_deb in fetch_ret['deb-failed']:
logger.error("Failed to download reused debs: %s", ','.join(fetch_ret['deb-failed']))
return False
@ -1164,6 +1228,13 @@ class BuildController():
# If the sharing mode is enabled
if not reclaim and self.attrs['reuse']:
if pkg_name in self.lists['never_reuse_pkgs']:
if status == 'DSC_NO_UPDATE':
logger.info("%s is forbidden to reuse, but no need to build locally again", pkg_name)
else:
logger.info("%s is forbidden to reuse and will be build locally later", pkg_name)
status = 'DSC_BUILD'
else:
# 'reuse' should be handled for either no '-c' or '-c -all'
if self.attrs['avoid'] or (self.attrs['build_all'] and not self.attrs['avoid']):
logger.debug("Comparing with the remote shared dsc cache for %s", build_type)
@ -1732,8 +1803,16 @@ class BuildController():
status, dsc_file = self.create_dsc(pkg_name, pkg_dir, reclaim=False, build_type=build_type)
if status == 'DSC_BUILD' and dsc_file:
logger.debug("dsc_file = %s" % dsc_file)
need_build[pkg_dir.strip()] = dsc_file
layer_pkgdir_dscs[pkg_dir.strip()] = dsc_file
# need_build will be passed to scan_all_depends() to get these depended packages
# Not checking 'build_done' stamp for package in need_build will cause the case
# the target package does not rebuild, but all its depended packages are forced
# to be rebuilt. Put the checking for 'build_done' stamp here to fix this issue
pkg_dir = pkg_dir.strip()
if not self.get_stamp(pkg_dir, dsc_file, build_type, 'build_done'):
need_build[pkg_dir] = dsc_file
else:
no_need_build[pkg_dir] = dsc_file
layer_pkgdir_dscs[pkg_dir] = dsc_file
fdsc_file.write(dsc_file + '\n')
if self.attrs['upload_source'] and not skip_dsc and self.kits['repo_mgr']:
self.upload_with_dsc(pkg_name, dsc_file, REPO_SOURCE)
@ -1758,8 +1837,11 @@ class BuildController():
else:
if status == 'DSC_NO_UPDATE':
logger.debug("Create_dsc return DSC_NO_UPDATE for %s", dsc_file)
layer_pkgdir_dscs[pkg_dir.strip()] = dsc_file
no_need_build[pkg_dir.strip()] = dsc_file
layer_pkgdir_dscs[pkg_dir] = dsc_file
if not self.get_stamp(pkg_dir, dsc_file, build_type, 'build_done'):
need_build[pkg_dir] = dsc_file
else:
no_need_build[pkg_dir] = dsc_file
fdsc_file.write(dsc_file + '\n')
# Find the dependency chain
@ -1868,6 +1950,10 @@ class BuildController():
if pkg in layer_pkgdir_dscs.keys():
target_pkgdir_dscs[pkg] = layer_pkgdir_dscs[pkg]
self.lists['real_build_' + build_type].append(pkg)
# no_need_build is returned by create_dsc, it just means
# that there is not any changes on dsc file but the build
# stamp of the 2nd phase may not exist, if it does not, it
# still needs to be built
target_pkgdir_dscs.update(no_need_build)
if fdsc_file:
@ -1985,8 +2071,12 @@ if __name__ == "__main__":
action='store_true')
parser.add_argument('-t', '--test', help="Run package tests during build",
action='store_true')
parser.add_argument('--reuse', help="Reuse the debs from STX_SHARED_REPO", action='store_true')
reuse_types = parser.add_mutually_exclusive_group()
reuse_types.add_argument('--reuse', help="Reuse the debs from STX_SHARED_REPO(no signed debs)", action='store_true')
reuse_types.add_argument('--reuse_maximum', help="Reuse all debs from STX_SHARED_REPO", action='store_true')
parser.add_argument('--dl_reused', help="Download reused debs to build directory", action='store_true', default=False)
parser.add_argument('--refresh_chroots', help="Force to fresh chroots before build", action='store_true')
parser.add_argument('--parallel', help="The number of parallel build tasks", type=int, default=DEFAULT_PARALLEL_TASKS)
parser.add_argument('--poll_interval', help="The interval to poll the build status", type=int, default=DEFAULT_POLL_INTERVAL)
@ -2007,7 +2097,7 @@ if __name__ == "__main__":
args = parser.parse_args()
if args.reuse:
if args.reuse or args.reuse_maximum:
if args.clean and args.packages:
logger.error("Reuse mode can not be used for the clean build of specific packages.");
sys.exit(1)
@ -2057,13 +2147,15 @@ if __name__ == "__main__":
if args.max_make_jobs:
build_controller.attrs['max_make_jobs'] = args.max_make_jobs
if args.reuse:
if args.reuse or args.reuse_maximum:
build_controller.attrs['reuse'] = True
if args.reuse_maximum:
build_controller.attrs['reuse_max'] = True
if args.dl_reused:
build_controller.attrs['dl_reused'] = True
else:
if args.dl_reused:
logger.error("option 'dl_reused' only valid if '--reuse' is enabled, quit")
logger.error("option 'dl_reused' only valid if '--reuse|--reuse_maximum' is enabled, quit")
sys.exit(1)
if args.packages:
packages = args.packages.strip().split(',')

16
build-tools/stx/debsentry.py
View File

@ -17,6 +17,22 @@ import os
import pickle
def get_pkg_by_deb(clue, debname, logger):
try:
with open(clue, 'rb') as fclue:
try:
debs = pickle.load(fclue)
for pkgname, subdebs in debs.items():
if debname in subdebs:
return pkgname
except (EOFError, ValueError, AttributeError, ImportError, IndexError, pickle.UnpicklingError) as e:
logger.error(str(e))
logger.warn(f"debs_entry:failed to load {clue}, return None")
except IOError:
logger.warn(f"debs_entry:{clue} does not exist")
return None
def get_subdebs(clue, package, logger):
try:
with open(clue, 'rb') as fclue:

32
build-tools/stx/repo_manage.py
View File

@ -106,29 +106,40 @@ class AptFetch():
raise Exception('apt cache init failed.')
# Download a binary package into downloaded folder
def fetch_deb(self, pkg_name, pkg_version=''):
def fetch_deb(self, pkg_name, pkg_version):
'''Download a binary package'''
if not pkg_name:
raise Exception('Binary package name empty')
ret = ''
if not pkg_name or not pkg_version:
return ret
self.logger.info("Current downloading:%s:%s", pkg_name, pkg_version)
destdir = os.path.join(self.workdir, 'downloads', 'binary')
self.aptlock.acquire()
pkg = self.aptcache.get(pkg_name)
try:
pkg = self.aptcache[pkg_name]
if not pkg:
self.aptlock.release()
raise Exception('Binary package "%s" was not found' % pkg_name)
self.logger.error("Failed to find binary package %s", pkg_name)
return ret
default_candidate = pkg.candidate
if pkg_version:
self.logger.debug("The default candidate is %s", default_candidate.version)
candidate = pkg.versions.get(pkg_version)
if not candidate:
if default_candidate:
if ':' in default_candidate.version:
epoch, ver = default_candidate.version.split(':')
if epoch.isdigit() and ver == pkg_version:
self.logger.debug('epoch %s will be skipped for %s_%s', epoch, pkg_name, ver)
candidate = default_candidate
else:
if not candidate:
self.aptlock.release()
raise Exception('Binary package "%s %s" was not found.' % (pkg_name, pkg_version))
self.logger.error("Failed to found the matched version %s for %s", pkg_version, pkg_name)
return ret
except Exception as e:
self.aptlock.release()
self.logger.error("Exception during candidate searching:%s", str(e))
return ret
uri = candidate.uri
filename = candidate.filename
self.aptlock.release()
@ -212,6 +223,9 @@ class AptFetch():
else:
pkg_version = pkg_ver.split()[1]
obj = threads.submit(self.fetch_deb, pkg_name, pkg_version)
if not obj:
self.logger.error("Failed to download %s:%s", pkg_name, pkg_version)
else:
obj_list.append(obj)
# Download source packages
for pkg_ver in dsc_set: