starlingx/root
Code Issues Proposed changes

build-pkgs: Fixed the deb packages missing issue after reuse

Browse Source 
The '--reuse|--reuse_maximum' feature mirrors the remote
shared repository and imports all debs from the mirror
to the local repository, there are no deb packages in the
local build directory for the reused packages which make
some tasks like building docker images and secure boot
signing fail for the missing deb packages.
This commit supports the below functions to fix the above
issues:
a. If '--dl_reused' option is enabled for option '--reuse'
or '--reuse_maximum', all the reused deb packages will be
downloaded to their local build directory.
b. 'never_reuse.lst' will be checked and the packages listed
in it will be built locally instead of reusing them if the
option '--reuse' is enabled. And it will be ignored if the
option '--reuse_maximum' is enabled.

Test Plan:
Pass: build-pkgs (make sure the normal build-pkgs works)
Pass: export STX_SHARED_REPO=<url to shard repo>
      export STX_SHARED_SOURCE=<url to shared source>
      build-pkgs --reuse
Pass: export STX_SHARED_REPO=<url to shard repo>
      export STX_SHARED_SOURCE=<url to shared source>
      build-pkgs --reuse --dl_reused
Pass: export STX_SHARED_REPO=<url to shard repo>
      export STX_SHARED_SOURCE=<url to shared source>
      build-pkgs --clean --reuse --dl_reused
      Run the secure boot signing script
Pass: export STX_SHARED_REPO=<url to shard repo>
      export STX_SHARED_SOURCE=<url to shared source>
      build-pkgs --clean --reuse --dl_reused
      build-pkgs (Make sure this build will not build from scratch)
Pass: export STX_SHARED_REPO=<url to shard repo>
      export STX_SHARED_SOURCE=<url to shared source>
      build-pkgs --clean --reuse_maximum --dl_reused
Pass: export STX_SHARED_REPO=<url to shard repo>
      export STX_SHARED_SOURCE=<url to shared source>
      build-pkgs --reuse_maximum --dl_reused

Partial-Bug: 2017763

Signed-off-by: hqbai <haiqing.bai@windriver.com>
Change-Id: I8cd84dbe6fe8f0262dde12befb0b16367e261968
This commit is contained in:
hqbai 2023-05-04 10:33:40 +08:00 committed by Haiqing Bai
parent 5c1e7e7b75
commit add2c84a28
3 changed files with 168 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
build-tools/stx/build-pkgs
View File

@ -457,6 +457,56 @@ def get_package_jobs(pkg_dir, distro=STX_DEFAULT_DISTRO):
return int(jobs)
def get_never_reuse_pkgs():
never_reuse_pkgs = set()
lst_dir = os.path.join(os.environ.get('MY_REPO_ROOT_DIR'),
'stx-tools/debian-mirror-tools/config/debian/common')
never_reuse_lst = os.path.join(lst_dir, 'never_reuse.lst')
try:
with open(never_reuse_lst, 'r') as npkgs:
lines = list(line for line in (p.strip() for p in npkgs) if line)
except Exception as e:
logger.warning(str(e))
return never_reuse_pkgs
else:
for pkg in lines:
pkg = pkg.strip()
if pkg.startswith('#'):
continue
never_reuse_pkgs.add(pkg)
return never_reuse_pkgs
def move_debs_to_build_dir(dl_bin_debs_dir):
try:
for root, dirs, files in os.walk(dl_bin_debs_dir):
if dirs:
pass
for r in files:
if r.endswith('.deb'):
pkg_item = r.split('_')
sdeb = '_'.join([pkg_item[0], pkg_item[1]])
pname = ''
for btype in ['std', 'rt']:
debs_clue = get_debs_clue(btype)
deb_file = os.path.join(root, r)
pname = debsentry.get_pkg_by_deb(debs_clue, sdeb, logger)
if pname:
pkg_build_dir = os.path.join(BUILD_ROOT, btype, pname)
os.makedirs(pkg_build_dir, exist_ok=True)
os.system('sudo rm -f %s/*.build' % (pkg_build_dir))
shutil.move(deb_file, os.path.join(pkg_build_dir, r))
logger.debug("Reuse: %s is moved to build directory", sdeb)
break
if not pname:
logger.warning("Failed to get the package name for %s", sdeb)
except Exception as e:
logger.error("An exception occurred during moving reused debs into build directory")
logger.error(str(e))
return False
return True
class repoSnapshots():
"""
The repository snapshots pool to manage the apply/release
@ -504,6 +554,7 @@ class BuildController():
'upload_source': False,
'poll_build_status': True,
'reuse': False,
'reuse_max': False,
'build_all': False,
'reuse_export': True,
'dl_reused': False,
@ -650,6 +701,9 @@ class BuildController():
os.makedirs(caches_dir, exist_ok=True)
self.lists['pkgs_not_found'] = []
self.lists['never_reuse_pkgs'] = []
if self.attrs['reuse'] and not self.attrs['reuse_max']:
self.lists['never_reuse_pkgs'] = get_never_reuse_pkgs()
for build_type in build_types_to_init:
self.lists['success_' + build_type] = []
@ -736,10 +790,17 @@ class BuildController():
def download_reused_debs(self, distribution):
if not self.attrs['dl_reused']:
return True
reuse_dl_dir = os.path.join(BUILD_ROOT, 'reused_debs')
os.makedirs(reuse_dl_dir, exist_ok=True)
apt_src_file = os.path.join(BUILD_ROOT, 'aptsrc')
try:
reuse_dl_dir = os.path.join(BUILD_ROOT, 'reused_debs')
if os.path.exists(reuse_dl_dir):
shutil.rmtree(reuse_dl_dir)
if os.path.exists(reuse_dl_dir):
logger.error("Failed to clean the old download directory")
logger.error("Please check and make sure it is removed")
return False
os.makedirs(reuse_dl_dir, exist_ok=True)
apt_src_file = os.path.join(BUILD_ROOT, 'aptsrc')
with open(apt_src_file, 'w') as f:
reuse_url = os.environ.get('STX_SHARED_REPO')
apt_item = ' '.join(['deb [trusted=yes]', reuse_url, distribution, 'main\n'])
@ -771,10 +832,13 @@ class BuildController():
return False
if len(fetch_ret['deb-failed']) == 0:
logger.info("Successfully downloaded all reused debs to %s", reuse_dl_dir + '/downloads/binary')
dl_bin_debs_dir=os.path.join(reuse_dl_dir, 'downloads/binary')
logger.info("Successfully downloaded all reused debs to %s", dl_bin_debs_dir)
move_debs_to_build_dir(dl_bin_debs_dir)
return True
else:
logger.error("Failed to download reused debs: %s", ','.join(fetch_ret['deb-failed']))
for failed_deb in fetch_ret['deb-failed']:
logger.error("Failed to download reused debs: %s", ','.join(fetch_ret['deb-failed']))
return False
def set_reuse(self, cache_dir):
@ -1164,28 +1228,35 @@ class BuildController():
# If the sharing mode is enabled
if not reclaim and self.attrs['reuse']:
# 'reuse' should be handled for either no '-c' or '-c -all'
if self.attrs['avoid'] or (self.attrs['build_all'] and not self.attrs['avoid']):
logger.debug("Comparing with the remote shared dsc cache for %s", build_type)
# Only match the subdir under STX REPO
pkg_stx_path = pkg_dir.replace(os.environ.get('MY_REPO'), '')
remote_dsc, shared_checksum = self.kits['dsc_rcache'][build_type].get_package_re(pkg_stx_path)
logger.debug("Checking package=%s, shared_checksum=%s, local_checksum=%s", pkg_stx_path, shared_checksum, new_checksum)
if shared_checksum and shared_checksum == new_checksum:
logger.debug("Same checksum, %s will be reused from remote", pkg_name)
# True None: just continue in the external for loop
status = 'DSC_REUSE'
'''
Here the local dsc_cache also need to be set which prevents the subsequent
build without 'reuse' rebuilding the package with same checksum again
'''
if dsc_file:
self.kits['dsc_cache'][build_type].set_package(pkg_dir, dsc_file + ':' + shared_checksum)
else:
logger.warning("dsc file is invalid and can not set dsc cache for %s", pkg_name)
if pkg_name in self.lists['never_reuse_pkgs']:
if status == 'DSC_NO_UPDATE':
logger.info("%s is forbidden to reuse, but no need to build locally again", pkg_name)
else:
logger.debug("Different source checksums, can not reuse the remote, continue to local build")
logger.info("%s is forbidden to reuse and will be build locally later", pkg_name)
status = 'DSC_BUILD'
else:
# 'reuse' should be handled for either no '-c' or '-c -all'
if self.attrs['avoid'] or (self.attrs['build_all'] and not self.attrs['avoid']):
logger.debug("Comparing with the remote shared dsc cache for %s", build_type)
# Only match the subdir under STX REPO
pkg_stx_path = pkg_dir.replace(os.environ.get('MY_REPO'), '')
remote_dsc, shared_checksum = self.kits['dsc_rcache'][build_type].get_package_re(pkg_stx_path)
logger.debug("Checking package=%s, shared_checksum=%s, local_checksum=%s", pkg_stx_path, shared_checksum, new_checksum)
if shared_checksum and shared_checksum == new_checksum:
logger.debug("Same checksum, %s will be reused from remote", pkg_name)
# True None: just continue in the external for loop
status = 'DSC_REUSE'
'''
Here the local dsc_cache also need to be set which prevents the subsequent
build without 'reuse' rebuilding the package with same checksum again
'''
if dsc_file:
self.kits['dsc_cache'][build_type].set_package(pkg_dir, dsc_file + ':' + shared_checksum)
else:
logger.warning("dsc file is invalid and can not set dsc cache for %s", pkg_name)
else:
logger.debug("Different source checksums, can not reuse the remote, continue to local build")
status = 'DSC_BUILD'
return status, dsc_file
@ -1732,8 +1803,16 @@ class BuildController():
status, dsc_file = self.create_dsc(pkg_name, pkg_dir, reclaim=False, build_type=build_type)
if status == 'DSC_BUILD' and dsc_file:
logger.debug("dsc_file = %s" % dsc_file)
need_build[pkg_dir.strip()] = dsc_file
layer_pkgdir_dscs[pkg_dir.strip()] = dsc_file
# need_build will be passed to scan_all_depends() to get these depended packages
# Not checking 'build_done' stamp for package in need_build will cause the case
# the target package does not rebuild, but all its depended packages are forced
# to be rebuilt. Put the checking for 'build_done' stamp here to fix this issue
pkg_dir = pkg_dir.strip()
if not self.get_stamp(pkg_dir, dsc_file, build_type, 'build_done'):
need_build[pkg_dir] = dsc_file
else:
no_need_build[pkg_dir] = dsc_file
layer_pkgdir_dscs[pkg_dir] = dsc_file
fdsc_file.write(dsc_file + '\n')
if self.attrs['upload_source'] and not skip_dsc and self.kits['repo_mgr']:
self.upload_with_dsc(pkg_name, dsc_file, REPO_SOURCE)
@ -1758,8 +1837,11 @@ class BuildController():
else:
if status == 'DSC_NO_UPDATE':
logger.debug("Create_dsc return DSC_NO_UPDATE for %s", dsc_file)
layer_pkgdir_dscs[pkg_dir.strip()] = dsc_file
no_need_build[pkg_dir.strip()] = dsc_file
layer_pkgdir_dscs[pkg_dir] = dsc_file
if not self.get_stamp(pkg_dir, dsc_file, build_type, 'build_done'):
need_build[pkg_dir] = dsc_file
else:
no_need_build[pkg_dir] = dsc_file
fdsc_file.write(dsc_file + '\n')
# Find the dependency chain
@ -1868,6 +1950,10 @@ class BuildController():
if pkg in layer_pkgdir_dscs.keys():
target_pkgdir_dscs[pkg] = layer_pkgdir_dscs[pkg]
self.lists['real_build_' + build_type].append(pkg)
# no_need_build is returned by create_dsc, it just means
# that there is not any changes on dsc file but the build
# stamp of the 2nd phase may not exist, if it does not, it
# still needs to be built
target_pkgdir_dscs.update(no_need_build)
if fdsc_file:
@ -1985,8 +2071,12 @@ if __name__ == "__main__":
action='store_true')
parser.add_argument('-t', '--test', help="Run package tests during build",
action='store_true')
parser.add_argument('--reuse', help="Reuse the debs from STX_SHARED_REPO", action='store_true')
reuse_types = parser.add_mutually_exclusive_group()
reuse_types.add_argument('--reuse', help="Reuse the debs from STX_SHARED_REPO(no signed debs)", action='store_true')
reuse_types.add_argument('--reuse_maximum', help="Reuse all debs from STX_SHARED_REPO", action='store_true')
parser.add_argument('--dl_reused', help="Download reused debs to build directory", action='store_true', default=False)
parser.add_argument('--refresh_chroots', help="Force to fresh chroots before build", action='store_true')
parser.add_argument('--parallel', help="The number of parallel build tasks", type=int, default=DEFAULT_PARALLEL_TASKS)
parser.add_argument('--poll_interval', help="The interval to poll the build status", type=int, default=DEFAULT_POLL_INTERVAL)
@ -2007,7 +2097,7 @@ if __name__ == "__main__":
args = parser.parse_args()
if args.reuse:
if args.reuse or args.reuse_maximum:
if args.clean and args.packages:
logger.error("Reuse mode can not be used for the clean build of specific packages.");
sys.exit(1)
@ -2057,13 +2147,15 @@ if __name__ == "__main__":
if args.max_make_jobs:
build_controller.attrs['max_make_jobs'] = args.max_make_jobs
if args.reuse:
if args.reuse or args.reuse_maximum:
build_controller.attrs['reuse'] = True
if args.reuse_maximum:
build_controller.attrs['reuse_max'] = True
if args.dl_reused:
build_controller.attrs['dl_reused'] = True
else:
if args.dl_reused:
logger.error("option 'dl_reused' only valid if '--reuse' is enabled, quit")
logger.error("option 'dl_reused' only valid if '--reuse|--reuse_maximum' is enabled, quit")
sys.exit(1)
if args.packages:
packages = args.packages.strip().split(',')

16
build-tools/stx/debsentry.py
View File

@ -17,6 +17,22 @@ import os
import pickle
def get_pkg_by_deb(clue, debname, logger):
try:
with open(clue, 'rb') as fclue:
try:
debs = pickle.load(fclue)
for pkgname, subdebs in debs.items():
if debname in subdebs:
return pkgname
except (EOFError, ValueError, AttributeError, ImportError, IndexError, pickle.UnpicklingError) as e:
logger.error(str(e))
logger.warn(f"debs_entry:failed to load {clue}, return None")
except IOError:
logger.warn(f"debs_entry:{clue} does not exist")
return None
def get_subdebs(clue, package, logger):
try:
with open(clue, 'rb') as fclue:

40
build-tools/stx/repo_manage.py
View File

@ -106,29 +106,40 @@ class AptFetch():
raise Exception('apt cache init failed.')
# Download a binary package into downloaded folder
def fetch_deb(self, pkg_name, pkg_version=''):
def fetch_deb(self, pkg_name, pkg_version):
'''Download a binary package'''
if not pkg_name:
raise Exception('Binary package name empty')
ret = ''
if not pkg_name or not pkg_version:
return ret
self.logger.info("Current downloading:%s:%s", pkg_name, pkg_version)
destdir = os.path.join(self.workdir, 'downloads', 'binary')
self.aptlock.acquire()
pkg = self.aptcache.get(pkg_name)
if not pkg:
self.aptlock.release()
raise Exception('Binary package "%s" was not found' % pkg_name)
default_candidate = pkg.candidate
if pkg_version:
try:
pkg = self.aptcache[pkg_name]
if not pkg:
self.aptlock.release()
self.logger.error("Failed to find binary package %s", pkg_name)
return ret
default_candidate = pkg.candidate
self.logger.debug("The default candidate is %s", default_candidate.version)
candidate = pkg.versions.get(pkg_version)
if not candidate:
if default_candidate:
if ':' in default_candidate.version:
epoch, ver = default_candidate.version.split(':')
if epoch.isdigit() and ver == pkg_version:
self.logger.debug('epoch %s will be skipped for %s_%s', epoch, pkg_name, ver)
candidate = default_candidate
else:
if not candidate:
self.aptlock.release()
raise Exception('Binary package "%s %s" was not found.' % (pkg_name, pkg_version))
self.logger.error("Failed to found the matched version %s for %s", pkg_version, pkg_name)
return ret
except Exception as e:
self.aptlock.release()
self.logger.error("Exception during candidate searching:%s", str(e))
return ret
uri = candidate.uri
filename = candidate.filename
self.aptlock.release()
@ -212,7 +223,10 @@ class AptFetch():
else:
pkg_version = pkg_ver.split()[1]
obj = threads.submit(self.fetch_deb, pkg_name, pkg_version)
obj_list.append(obj)
if not obj:
self.logger.error("Failed to download %s:%s", pkg_name, pkg_version)
else:
obj_list.append(obj)
# Download source packages
for pkg_ver in dsc_set:
pkg_name = pkg_ver.split()[0]