From 85ecb712be91d6063c13cf47b8ed165e9a9af7ee Mon Sep 17 00:00:00 2001 From: Davlet Panech Date: Tue, 5 Jul 2022 11:10:29 -0400 Subject: [PATCH] build-tools: apt repo priority based on "Origin" * build-docker-images/stx-debian/stx.preferences.part.in * build-docker-images/build-base-image.sh This file sets base image priorities for apt repos that match certain properties. Formerly we used higher priority for repos hosted by the build server, as opposed to debian.org or others. This doesn't work when using a Debian mirror hosted on the build server itself, since the hostname of both repos are equal. Solution: increase priority for repos whose "Release" file contains the field "Origin: $REPOMGR_ORIGIN" and make sure aptly adds that field to its Release file. The value comes from the environment and should be set by the build container. * stx/aptly_deb_usage.py: Add an "Origin" field to non-mirror publications, value taken from environment REPOMGR_ORIGIN * build-docker-images/stx-debian/Dockerfile.stable Improvements to package conflict resolution and docker FS caching-related issues: - Upgrade base packages to versions in managed repos before doing anything else - Install packages provided by upstream debian in a separate RUN command/docker FS layer - Make sure each "apt-get install" is in its own RUN command and is preceded with "apt-get update" -- to avoid using stale metadata due to "docker build" FS layer caching TESTS ====================== - Define REPOMGR_ORIGIN in container environment - Run downloader & build-pkgs & make sure generated repos' Release file contains "Origin: starlingx" - Build base image & make sure its apt.preferences contains the priority rule for "Origin: starlingx" Story: 2010055 Task: 45729 Change-Id: Ibaafbfbeef408904d216265168daa466d90fc7f2 Signed-off-by: Davlet Panech --- .../build-docker-images/build-stx-base.sh | 14 +--- .../stx-debian/Dockerfile.stable | 82 +++++++++++++------ .../stx-debian/apt/stx.preferences.part.in | 4 +- build-tools/stx/aptly_deb_usage.py | 11 ++- build-tools/stx/build-image | 3 +- build-tools/stx/build-pkgs | 4 +- build-tools/stx/debdownloader | 3 +- build-tools/stx/downloader | 3 +- build-tools/stx/repo_manage.py | 27 +++--- 9 files changed, 97 insertions(+), 54 deletions(-) diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index 1582aac7..441b8d1d 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -333,7 +333,7 @@ EOF else # These env vars must be defined in debian builder pods - for var in DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION REPOMGR_DEPLOY_URL ; do + for var in DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION REPOMGR_DEPLOY_URL REPOMGR_ORIGIN ; do if [[ -z "${!var}" ]] ; then echo "$var must be defined in the environment!" >&2 exit 1 @@ -347,7 +347,7 @@ else -e "s!@DEBIAN_SECURITY_SNAPSHOT@!${DEBIAN_SECURITY_SNAPSHOT}!g" \ -e "s!@DEBIAN_DISTRIBUTION@!${DEBIAN_DISTRIBUTION}!g" \ -e "s!@REPOMGR_DEPLOY_URL@!${REPOMGR_DEPLOY_URL}!g" \ - -e "s!@REPOMGR_HOST@!${REPOMGR_HOST}!g" \ + -e "s!@REPOMGR_ORIGIN@!${REPOMGR_ORIGIN}!g" \ "$@" } @@ -369,14 +369,8 @@ else replace_vars "${SRC_DOCKER_DIR}/apt/stx.sources.list.in" >"${BUILDDIR}/apt/stx.sources.list" fi - # preferences: instantiate template once for every host in stx.sources.list - unique_hosts=$(\grep -v -E '^\s*(#.*)?$' "${BUILDDIR}/apt/stx.sources.list" | sed -n -r 's#.*(https?|ftp)://([^/:[:space:]]+).*#\2#p' | sort -u) - echo -n >"${BUILDDIR}/apt/stx.preferences" - for host in $unique_hosts ; do - REPOMGR_HOST="$host" replace_vars "${SRC_DOCKER_DIR}/apt/stx.preferences.part.in" >>"${BUILDDIR}/apt/stx.preferences" - echo >>"${BUILDDIR}/apt/stx.preferences" - done - unset host unique_hosts + # preferences: instantiate template with REPOMGR_ORIGIN from environment + replace_vars "${SRC_DOCKER_DIR}/apt/stx.preferences.part.in" >>"${BUILDDIR}/apt/stx.preferences" unset -f replace_vars fi diff --git a/build-tools/build-docker-images/stx-debian/Dockerfile.stable b/build-tools/build-docker-images/stx-debian/Dockerfile.stable index 5b0acfd7..aee50048 100644 --- a/build-tools/build-docker-images/stx-debian/Dockerfile.stable +++ b/build-tools/build-docker-images/stx-debian/Dockerfile.stable @@ -18,17 +18,6 @@ COPY apt/debian.sources.list /etc/apt/sources.list.d/debian.list.disabled COPY apt/stx.sources.list /etc/apt/sources.list.d/stx.list.disabled COPY apt/stx.preferences /etc/apt/preferences.d/stx -# Enable stx repo -RUN cp /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list - -# Clean apt cache -RUN apt-get clean && rm -rf /var/lib/apt/lists/* - -# Upgrade base packages to versions in the managed repos -RUN apt-get -y update && \ - apt-get -y upgrade && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - # repo templates: # /etc/apt/sources.list.d/ # debian.list.disabled - vanilla debian repos @@ -48,18 +37,64 @@ RUN apt-get -y update && \ # Enabling the upstream repos ("debian.list") is dangerous because it # may conflict with packages in stx.list. # +# +# FIXME: apt evaluates these files in alphabetical order, so stx.list +# comes after debian.list. When the local binary repo contains +# the same package/version as the debian repo, apt will download +# it from debian, regardless of the priority in /etc/apt/preferences. +# We should rename these files to make stx.list sort before +# debian.list. This would affect Loci scripts in +# loci/docker/stx-scripts/ +# +# +# Upgrade base packages to versions in managed repos +# +RUN cp -f /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list && \ + apt-get -y update && \ + apt-get -y upgrade && \ + rm -f /etc/apt/sources.list.d/stx.list && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# +# Install packages provided only by debian. +# FIXME: move these packages + their dependencies to debian download lists in +# starlingx/tools to avoid referencing the debian repo at all. +# +RUN cp -f /etc/apt/sources.list.d/debian.list.disabled /etc/apt/sources.list.d/debian.list && \ + cp -f /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list && \ + apt-get update -y && \ + apt-get install -y \ + libapache2-mod-wsgi-py3 \ + python3-setuptools \ + build-essential \ + && \ + rm -f /etc/apt/sources.list.d/debian.list && \ + rm -f /etc/apt/sources.list.d/stx.list && \ + apt-get clean && \ + rm -rf /var/lib/apt/files/* + +# +# Enable stx repo only. Packages installs below this point will use +# only the managed locally-built & 3rd-party repos. +# +RUN cp /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list + +# +# Install required packages +# RUN apt-get update -y && \ apt-get upgrade -y && \ apt-get install -y \ -# FIXME: uncomment once qemu is ported to debian (starlingx/integ) -# qemu-utils \ openssh-client \ python3 \ python3-pip \ python3-wheel \ - libapache2-mod-wsgi-py3 \ - ; +# FIXME: uncomment once qemu is ported to debian (starlingx/integ) +# qemu-utils \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/files/* # FIXME: these packages are not required by most docker images inheriting # from this image. However these Python modules are not buildable from @@ -69,11 +104,12 @@ RUN apt-get update -y && \ # # A better solution would be to omit them here, but install them in each # project that requires them; or add wheel subpackages to these DEBs. -RUN apt-get install -y \ - python3-thriftpy \ - python3-nss \ - python-nss - -# Delete apt cache -RUN apt-get clean && \ - rm -rf /var/lib/apt/lists/* +RUN apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install -y \ + python3-thriftpy \ + python3-nss \ + python-nss \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/files/* diff --git a/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in b/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in index 5863a2e0..367e3d99 100644 --- a/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in +++ b/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in @@ -1,4 +1,4 @@ -Explanation: Prefer StarlingX repos over vanilla Debian +Explanation: Prefer StarlingX repos over anything else Package: * -Pin: origin "@REPOMGR_HOST@" +Pin: release o=@REPOMGR_ORIGIN@ Pin-Priority: 999 diff --git a/build-tools/stx/aptly_deb_usage.py b/build-tools/stx/aptly_deb_usage.py index 80554cc1..5bb10fed 100755 --- a/build-tools/stx/aptly_deb_usage.py +++ b/build-tools/stx/aptly_deb_usage.py @@ -47,12 +47,16 @@ SIGN_PASSWD = 'starlingx' class Deb_aptly(): - def __init__(self, url, logger): + def __init__(self, url, origin, logger): '''The basic interface to manage aptly database. ''' self.logger = logger self.url = url self.aptly = Client(self.url) self.logger.info('Aptly connected, version: %s', self.aptly.misc.version) + if origin: + self.origin = origin.strip() or None + else: + self.origin = None # Create a remote mirror(make sure the name has specified prefix) # Input @@ -291,10 +295,12 @@ class Deb_aptly(): # Add 'source' to publish source packages, if no source packages, that is also harmless. extra_param['architectures'] = mirror.architectures.append('source') extra_param['distribution'] = mirror.distribution + extra_param['origin'] = None else: # Only support binary_amd64 and source packages extra_param['architectures'] = ['amd64', 'source'] extra_param['distribution'] = None + extra_param['origin'] = self.origin extra_param['source_kind'] = 'snapshot' extra_param['sources'] = [{'Name': name}] @@ -304,7 +310,8 @@ class Deb_aptly(): task = self.aptly.publish.publish(source_kind='snapshot', sources=extra_param['sources'], architectures=extra_param['architectures'], prefix=extra_param['prefix'], distribution=extra_param['distribution'], - sign_gpgkey=SIGN_KEY, sign_passphrase=SIGN_PASSWD) + sign_gpgkey=SIGN_KEY, sign_passphrase=SIGN_PASSWD, + origin=extra_param['origin']) # In corner cases, "publish" may spend more than 60 seconds, cause # "wait_for_task_by_id" timeout. To cover such cases, we "wait_for_task_by_id" # up to 3 times, to enlarge the whole timeout to 180 seconds. diff --git a/build-tools/stx/build-image b/build-tools/stx/build-image index 1d7192d3..92869d22 100755 --- a/build-tools/stx/build-image +++ b/build-tools/stx/build-image @@ -435,7 +435,8 @@ if __name__ == "__main__": rmg_logger = logging.getLogger('repo_manager') utils.set_logger(rmg_logger) repo_manager = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'), - '/tmp/', rmg_logger) + '/tmp/', os.environ.get('REPOMGR_ORIGIN'), + rmg_logger) repo_manager.upload_pkg(REPO_BUILD, None) repo_manager.upload_pkg(REPO_BINARY, None) diff --git a/build-tools/stx/build-pkgs b/build-tools/stx/build-pkgs index 059ad036..e78edfd2 100755 --- a/build-tools/stx/build-pkgs +++ b/build-tools/stx/build-pkgs @@ -35,6 +35,7 @@ import yaml BUILDER_URL = os.environ.get('BUILDER_URL') REPOMGR_URL = os.environ.get('REPOMGR_URL') +REPOMGR_ORIGIN = os.environ.get('REPOMGR_ORIGIN') BUILD_ROOT = os.environ.get('MY_BUILD_PKG_DIR') STX_ROOT = os.environ.get('MY_REPO_ROOT_DIR') PKGBUILDER_ROOT = "/localdisk/pkgbuilder" @@ -274,7 +275,8 @@ class BuildController(): rlogger = logging.getLogger('repo_manager') utils.set_logger(rlogger) self.kits['repo_mgr'] = repo_manage.RepoMgr('aptly', REPOMGR_URL, - '/tmp', rlogger) + '/tmp', REPOMGR_ORIGIN, + rlogger) logger.debug("Successful created repo manager") @property diff --git a/build-tools/stx/debdownloader b/build-tools/stx/debdownloader index 7a26b90a..6988cc73 100755 --- a/build-tools/stx/debdownloader +++ b/build-tools/stx/debdownloader @@ -87,7 +87,8 @@ if __name__ == "__main__": # Create local binary repo with repo manager repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'), - '/tmp/', logger) + '/tmp/', os.environ.get('REPOMGR_ORIGIN'), + logger) repomgr.upload_pkg(REPO_BIN, None) with open(sys.argv[1], 'r') as flist: diff --git a/build-tools/stx/downloader b/build-tools/stx/downloader index e5c4b570..1e309bf3 100755 --- a/build-tools/stx/downloader +++ b/build-tools/stx/downloader @@ -165,7 +165,8 @@ class BaseDownloader(): rlogger = logging.getLogger('repo_manager') utils.set_logger(rlogger) self.repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'), - '/tmp/', rlogger) + '/tmp/', os.environ.get('REPOMGR_ORIGIN'), + rlogger) def clean(self): if os.path.exists(self.dl_dir): diff --git a/build-tools/stx/repo_manage.py b/build-tools/stx/repo_manage.py index b22bec76..761cdc8b 100755 --- a/build-tools/stx/repo_manage.py +++ b/build-tools/stx/repo_manage.py @@ -33,6 +33,7 @@ import utils REPOMGR_URL = os.environ.get('REPOMGR_URL') +REPOMGR_ORIGIN = os.environ.get('REPOMGR_ORIGIN') REPOMGR_DEPLOY_URL = os.environ.get('REPOMGR_DEPLOY_URL') APTFETCH_JOBS = 10 @@ -218,9 +219,9 @@ class RepoMgr(): remote repo: mirror of another repository. shouldn't insert or remove packages from it.. local repo: a local repository, we can insert/remove packages into/from them. ''' - def __init__(self, repoType, repoURL, workdir, logger): + def __init__(self, repoType, repoURL, workdir, origin, logger): if repoType == 'aptly': - self.repo = aptly_deb_usage.Deb_aptly(repoURL, logger) + self.repo = aptly_deb_usage.Deb_aptly(repoURL, origin, logger) else: raise Exception('Currently, only aptly repository supported') @@ -662,44 +663,44 @@ utils.set_logger(applogger) def _handleDownload(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, REPOMGR_ORIGIN, applogger) kwargs = {'sources_list': args.sources_list, 'deb_list': args.deb_list, 'dsc_list': args.dsc_list} repomgr.download(args.name, **kwargs, no_clear=args.no_clear) def _handleSync(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, REPOMGR_ORIGIN, applogger) kwargs = {'sources_list': args.sources_list, 'deb_list': args.deb_list, 'dsc_list': args.dsc_list} repomgr.sync(args.name, args.repo_list, **kwargs, no_clear=args.no_clear) def _handleMirror(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) kwargs = {'url': args.url, 'distribution': args.distribution, 'component': args.component, 'architectures': args.architectures, 'with_sources': args.with_sources} repomgr.mirror(args.name, **kwargs) def _handleMerge(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.merge(args.name, args.repo_list) def _handleUploadPkg(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.upload_pkg(args.repository, args.package) def _handleDeletePkg(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.delete_pkg(args.repository, args.package_name, args.package_type, pkg_version=args.package_version) def _handleSearchPkg(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) if args.package_type == 'binary': repomgr.search_pkg(args.repository, args.package_name, pkg_version=args.package_version, binary=True) @@ -709,22 +710,22 @@ def _handleSearchPkg(args): def _handleRemoveRope(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.remove_repo(args.repository) def _handleList(_args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.list() def _handleListPkgs(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.list_pkgs(args.repository) def _handleClean(_args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.clean()