From add2c84a28ed7b371c0c20ab1be1a55e0ca7410f Mon Sep 17 00:00:00 2001 From: hqbai Date: Thu, 4 May 2023 10:33:40 +0800 Subject: [PATCH] build-pkgs: Fixed the deb packages missing issue after reuse The '--reuse|--reuse_maximum' feature mirrors the remote shared repository and imports all debs from the mirror to the local repository, there are no deb packages in the local build directory for the reused packages which make some tasks like building docker images and secure boot signing fail for the missing deb packages. This commit supports the below functions to fix the above issues: a. If '--dl_reused' option is enabled for option '--reuse' or '--reuse_maximum', all the reused deb packages will be downloaded to their local build directory. b. 'never_reuse.lst' will be checked and the packages listed in it will be built locally instead of reusing them if the option '--reuse' is enabled. And it will be ignored if the option '--reuse_maximum' is enabled. Test Plan: Pass: build-pkgs (make sure the normal build-pkgs works) Pass: export STX_SHARED_REPO= export STX_SHARED_SOURCE= build-pkgs --reuse Pass: export STX_SHARED_REPO= export STX_SHARED_SOURCE= build-pkgs --reuse --dl_reused Pass: export STX_SHARED_REPO= export STX_SHARED_SOURCE= build-pkgs --clean --reuse --dl_reused Run the secure boot signing script Pass: export STX_SHARED_REPO= export STX_SHARED_SOURCE= build-pkgs --clean --reuse --dl_reused build-pkgs (Make sure this build will not build from scratch) Pass: export STX_SHARED_REPO= export STX_SHARED_SOURCE= build-pkgs --clean --reuse_maximum --dl_reused Pass: export STX_SHARED_REPO= export STX_SHARED_SOURCE= build-pkgs --reuse_maximum --dl_reused Partial-Bug: 2017763 Signed-off-by: hqbai Change-Id: I8cd84dbe6fe8f0262dde12befb0b16367e261968 --- build-tools/stx/build-pkgs | 158 ++++++++++++++++++++++++++------- build-tools/stx/debsentry.py | 16 ++++ build-tools/stx/repo_manage.py | 40 ++++++--- 3 files changed, 168 insertions(+), 46 deletions(-) diff --git a/build-tools/stx/build-pkgs b/build-tools/stx/build-pkgs index c36d5a9b..5a4b3389 100755 --- a/build-tools/stx/build-pkgs +++ b/build-tools/stx/build-pkgs @@ -457,6 +457,56 @@ def get_package_jobs(pkg_dir, distro=STX_DEFAULT_DISTRO): return int(jobs) +def get_never_reuse_pkgs(): + never_reuse_pkgs = set() + lst_dir = os.path.join(os.environ.get('MY_REPO_ROOT_DIR'), + 'stx-tools/debian-mirror-tools/config/debian/common') + never_reuse_lst = os.path.join(lst_dir, 'never_reuse.lst') + try: + with open(never_reuse_lst, 'r') as npkgs: + lines = list(line for line in (p.strip() for p in npkgs) if line) + except Exception as e: + logger.warning(str(e)) + return never_reuse_pkgs + else: + for pkg in lines: + pkg = pkg.strip() + if pkg.startswith('#'): + continue + never_reuse_pkgs.add(pkg) + return never_reuse_pkgs + + +def move_debs_to_build_dir(dl_bin_debs_dir): + try: + for root, dirs, files in os.walk(dl_bin_debs_dir): + if dirs: + pass + for r in files: + if r.endswith('.deb'): + pkg_item = r.split('_') + sdeb = '_'.join([pkg_item[0], pkg_item[1]]) + pname = '' + for btype in ['std', 'rt']: + debs_clue = get_debs_clue(btype) + deb_file = os.path.join(root, r) + pname = debsentry.get_pkg_by_deb(debs_clue, sdeb, logger) + if pname: + pkg_build_dir = os.path.join(BUILD_ROOT, btype, pname) + os.makedirs(pkg_build_dir, exist_ok=True) + os.system('sudo rm -f %s/*.build' % (pkg_build_dir)) + shutil.move(deb_file, os.path.join(pkg_build_dir, r)) + logger.debug("Reuse: %s is moved to build directory", sdeb) + break + if not pname: + logger.warning("Failed to get the package name for %s", sdeb) + except Exception as e: + logger.error("An exception occurred during moving reused debs into build directory") + logger.error(str(e)) + return False + return True + + class repoSnapshots(): """ The repository snapshots pool to manage the apply/release @@ -504,6 +554,7 @@ class BuildController(): 'upload_source': False, 'poll_build_status': True, 'reuse': False, + 'reuse_max': False, 'build_all': False, 'reuse_export': True, 'dl_reused': False, @@ -650,6 +701,9 @@ class BuildController(): os.makedirs(caches_dir, exist_ok=True) self.lists['pkgs_not_found'] = [] + self.lists['never_reuse_pkgs'] = [] + if self.attrs['reuse'] and not self.attrs['reuse_max']: + self.lists['never_reuse_pkgs'] = get_never_reuse_pkgs() for build_type in build_types_to_init: self.lists['success_' + build_type] = [] @@ -736,10 +790,17 @@ class BuildController(): def download_reused_debs(self, distribution): if not self.attrs['dl_reused']: return True - reuse_dl_dir = os.path.join(BUILD_ROOT, 'reused_debs') - os.makedirs(reuse_dl_dir, exist_ok=True) - apt_src_file = os.path.join(BUILD_ROOT, 'aptsrc') + try: + reuse_dl_dir = os.path.join(BUILD_ROOT, 'reused_debs') + if os.path.exists(reuse_dl_dir): + shutil.rmtree(reuse_dl_dir) + if os.path.exists(reuse_dl_dir): + logger.error("Failed to clean the old download directory") + logger.error("Please check and make sure it is removed") + return False + os.makedirs(reuse_dl_dir, exist_ok=True) + apt_src_file = os.path.join(BUILD_ROOT, 'aptsrc') with open(apt_src_file, 'w') as f: reuse_url = os.environ.get('STX_SHARED_REPO') apt_item = ' '.join(['deb [trusted=yes]', reuse_url, distribution, 'main\n']) @@ -771,10 +832,13 @@ class BuildController(): return False if len(fetch_ret['deb-failed']) == 0: - logger.info("Successfully downloaded all reused debs to %s", reuse_dl_dir + '/downloads/binary') + dl_bin_debs_dir=os.path.join(reuse_dl_dir, 'downloads/binary') + logger.info("Successfully downloaded all reused debs to %s", dl_bin_debs_dir) + move_debs_to_build_dir(dl_bin_debs_dir) return True else: - logger.error("Failed to download reused debs: %s", ','.join(fetch_ret['deb-failed'])) + for failed_deb in fetch_ret['deb-failed']: + logger.error("Failed to download reused debs: %s", ','.join(fetch_ret['deb-failed'])) return False def set_reuse(self, cache_dir): @@ -1164,28 +1228,35 @@ class BuildController(): # If the sharing mode is enabled if not reclaim and self.attrs['reuse']: - # 'reuse' should be handled for either no '-c' or '-c -all' - if self.attrs['avoid'] or (self.attrs['build_all'] and not self.attrs['avoid']): - logger.debug("Comparing with the remote shared dsc cache for %s", build_type) - # Only match the subdir under STX REPO - pkg_stx_path = pkg_dir.replace(os.environ.get('MY_REPO'), '') - remote_dsc, shared_checksum = self.kits['dsc_rcache'][build_type].get_package_re(pkg_stx_path) - logger.debug("Checking package=%s, shared_checksum=%s, local_checksum=%s", pkg_stx_path, shared_checksum, new_checksum) - if shared_checksum and shared_checksum == new_checksum: - logger.debug("Same checksum, %s will be reused from remote", pkg_name) - # True None: just continue in the external for loop - status = 'DSC_REUSE' - ''' - Here the local dsc_cache also need to be set which prevents the subsequent - build without 'reuse' rebuilding the package with same checksum again - ''' - if dsc_file: - self.kits['dsc_cache'][build_type].set_package(pkg_dir, dsc_file + ':' + shared_checksum) - else: - logger.warning("dsc file is invalid and can not set dsc cache for %s", pkg_name) + if pkg_name in self.lists['never_reuse_pkgs']: + if status == 'DSC_NO_UPDATE': + logger.info("%s is forbidden to reuse, but no need to build locally again", pkg_name) else: - logger.debug("Different source checksums, can not reuse the remote, continue to local build") + logger.info("%s is forbidden to reuse and will be build locally later", pkg_name) status = 'DSC_BUILD' + else: + # 'reuse' should be handled for either no '-c' or '-c -all' + if self.attrs['avoid'] or (self.attrs['build_all'] and not self.attrs['avoid']): + logger.debug("Comparing with the remote shared dsc cache for %s", build_type) + # Only match the subdir under STX REPO + pkg_stx_path = pkg_dir.replace(os.environ.get('MY_REPO'), '') + remote_dsc, shared_checksum = self.kits['dsc_rcache'][build_type].get_package_re(pkg_stx_path) + logger.debug("Checking package=%s, shared_checksum=%s, local_checksum=%s", pkg_stx_path, shared_checksum, new_checksum) + if shared_checksum and shared_checksum == new_checksum: + logger.debug("Same checksum, %s will be reused from remote", pkg_name) + # True None: just continue in the external for loop + status = 'DSC_REUSE' + ''' + Here the local dsc_cache also need to be set which prevents the subsequent + build without 'reuse' rebuilding the package with same checksum again + ''' + if dsc_file: + self.kits['dsc_cache'][build_type].set_package(pkg_dir, dsc_file + ':' + shared_checksum) + else: + logger.warning("dsc file is invalid and can not set dsc cache for %s", pkg_name) + else: + logger.debug("Different source checksums, can not reuse the remote, continue to local build") + status = 'DSC_BUILD' return status, dsc_file @@ -1732,8 +1803,16 @@ class BuildController(): status, dsc_file = self.create_dsc(pkg_name, pkg_dir, reclaim=False, build_type=build_type) if status == 'DSC_BUILD' and dsc_file: logger.debug("dsc_file = %s" % dsc_file) - need_build[pkg_dir.strip()] = dsc_file - layer_pkgdir_dscs[pkg_dir.strip()] = dsc_file + # need_build will be passed to scan_all_depends() to get these depended packages + # Not checking 'build_done' stamp for package in need_build will cause the case + # the target package does not rebuild, but all its depended packages are forced + # to be rebuilt. Put the checking for 'build_done' stamp here to fix this issue + pkg_dir = pkg_dir.strip() + if not self.get_stamp(pkg_dir, dsc_file, build_type, 'build_done'): + need_build[pkg_dir] = dsc_file + else: + no_need_build[pkg_dir] = dsc_file + layer_pkgdir_dscs[pkg_dir] = dsc_file fdsc_file.write(dsc_file + '\n') if self.attrs['upload_source'] and not skip_dsc and self.kits['repo_mgr']: self.upload_with_dsc(pkg_name, dsc_file, REPO_SOURCE) @@ -1758,8 +1837,11 @@ class BuildController(): else: if status == 'DSC_NO_UPDATE': logger.debug("Create_dsc return DSC_NO_UPDATE for %s", dsc_file) - layer_pkgdir_dscs[pkg_dir.strip()] = dsc_file - no_need_build[pkg_dir.strip()] = dsc_file + layer_pkgdir_dscs[pkg_dir] = dsc_file + if not self.get_stamp(pkg_dir, dsc_file, build_type, 'build_done'): + need_build[pkg_dir] = dsc_file + else: + no_need_build[pkg_dir] = dsc_file fdsc_file.write(dsc_file + '\n') # Find the dependency chain @@ -1868,6 +1950,10 @@ class BuildController(): if pkg in layer_pkgdir_dscs.keys(): target_pkgdir_dscs[pkg] = layer_pkgdir_dscs[pkg] self.lists['real_build_' + build_type].append(pkg) + # no_need_build is returned by create_dsc, it just means + # that there is not any changes on dsc file but the build + # stamp of the 2nd phase may not exist, if it does not, it + # still needs to be built target_pkgdir_dscs.update(no_need_build) if fdsc_file: @@ -1985,8 +2071,12 @@ if __name__ == "__main__": action='store_true') parser.add_argument('-t', '--test', help="Run package tests during build", action='store_true') - parser.add_argument('--reuse', help="Reuse the debs from STX_SHARED_REPO", action='store_true') + + reuse_types = parser.add_mutually_exclusive_group() + reuse_types.add_argument('--reuse', help="Reuse the debs from STX_SHARED_REPO(no signed debs)", action='store_true') + reuse_types.add_argument('--reuse_maximum', help="Reuse all debs from STX_SHARED_REPO", action='store_true') parser.add_argument('--dl_reused', help="Download reused debs to build directory", action='store_true', default=False) + parser.add_argument('--refresh_chroots', help="Force to fresh chroots before build", action='store_true') parser.add_argument('--parallel', help="The number of parallel build tasks", type=int, default=DEFAULT_PARALLEL_TASKS) parser.add_argument('--poll_interval', help="The interval to poll the build status", type=int, default=DEFAULT_POLL_INTERVAL) @@ -2007,7 +2097,7 @@ if __name__ == "__main__": args = parser.parse_args() - if args.reuse: + if args.reuse or args.reuse_maximum: if args.clean and args.packages: logger.error("Reuse mode can not be used for the clean build of specific packages."); sys.exit(1) @@ -2057,13 +2147,15 @@ if __name__ == "__main__": if args.max_make_jobs: build_controller.attrs['max_make_jobs'] = args.max_make_jobs - if args.reuse: + if args.reuse or args.reuse_maximum: build_controller.attrs['reuse'] = True + if args.reuse_maximum: + build_controller.attrs['reuse_max'] = True if args.dl_reused: build_controller.attrs['dl_reused'] = True else: if args.dl_reused: - logger.error("option 'dl_reused' only valid if '--reuse' is enabled, quit") + logger.error("option 'dl_reused' only valid if '--reuse|--reuse_maximum' is enabled, quit") sys.exit(1) if args.packages: packages = args.packages.strip().split(',') diff --git a/build-tools/stx/debsentry.py b/build-tools/stx/debsentry.py index 0392ddd5..2dcada2f 100755 --- a/build-tools/stx/debsentry.py +++ b/build-tools/stx/debsentry.py @@ -17,6 +17,22 @@ import os import pickle +def get_pkg_by_deb(clue, debname, logger): + try: + with open(clue, 'rb') as fclue: + try: + debs = pickle.load(fclue) + for pkgname, subdebs in debs.items(): + if debname in subdebs: + return pkgname + except (EOFError, ValueError, AttributeError, ImportError, IndexError, pickle.UnpicklingError) as e: + logger.error(str(e)) + logger.warn(f"debs_entry:failed to load {clue}, return None") + except IOError: + logger.warn(f"debs_entry:{clue} does not exist") + return None + + def get_subdebs(clue, package, logger): try: with open(clue, 'rb') as fclue: diff --git a/build-tools/stx/repo_manage.py b/build-tools/stx/repo_manage.py index b6ba39fa..202535f4 100755 --- a/build-tools/stx/repo_manage.py +++ b/build-tools/stx/repo_manage.py @@ -106,29 +106,40 @@ class AptFetch(): raise Exception('apt cache init failed.') # Download a binary package into downloaded folder - def fetch_deb(self, pkg_name, pkg_version=''): + def fetch_deb(self, pkg_name, pkg_version): '''Download a binary package''' - if not pkg_name: - raise Exception('Binary package name empty') + ret = '' + if not pkg_name or not pkg_version: + return ret + self.logger.info("Current downloading:%s:%s", pkg_name, pkg_version) destdir = os.path.join(self.workdir, 'downloads', 'binary') self.aptlock.acquire() - pkg = self.aptcache.get(pkg_name) - if not pkg: - self.aptlock.release() - raise Exception('Binary package "%s" was not found' % pkg_name) - default_candidate = pkg.candidate - if pkg_version: + + try: + pkg = self.aptcache[pkg_name] + if not pkg: + self.aptlock.release() + self.logger.error("Failed to find binary package %s", pkg_name) + return ret + default_candidate = pkg.candidate + self.logger.debug("The default candidate is %s", default_candidate.version) candidate = pkg.versions.get(pkg_version) if not candidate: - if default_candidate: + if ':' in default_candidate.version: epoch, ver = default_candidate.version.split(':') if epoch.isdigit() and ver == pkg_version: self.logger.debug('epoch %s will be skipped for %s_%s', epoch, pkg_name, ver) candidate = default_candidate - else: + if not candidate: self.aptlock.release() - raise Exception('Binary package "%s %s" was not found.' % (pkg_name, pkg_version)) + self.logger.error("Failed to found the matched version %s for %s", pkg_version, pkg_name) + return ret + except Exception as e: + self.aptlock.release() + self.logger.error("Exception during candidate searching:%s", str(e)) + return ret + uri = candidate.uri filename = candidate.filename self.aptlock.release() @@ -212,7 +223,10 @@ class AptFetch(): else: pkg_version = pkg_ver.split()[1] obj = threads.submit(self.fetch_deb, pkg_name, pkg_version) - obj_list.append(obj) + if not obj: + self.logger.error("Failed to download %s:%s", pkg_name, pkg_version) + else: + obj_list.append(obj) # Download source packages for pkg_ver in dsc_set: pkg_name = pkg_ver.split()[0]