diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index 1582aac7..441b8d1d 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -333,7 +333,7 @@ EOF else # These env vars must be defined in debian builder pods - for var in DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION REPOMGR_DEPLOY_URL ; do + for var in DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION REPOMGR_DEPLOY_URL REPOMGR_ORIGIN ; do if [[ -z "${!var}" ]] ; then echo "$var must be defined in the environment!" >&2 exit 1 @@ -347,7 +347,7 @@ else -e "s!@DEBIAN_SECURITY_SNAPSHOT@!${DEBIAN_SECURITY_SNAPSHOT}!g" \ -e "s!@DEBIAN_DISTRIBUTION@!${DEBIAN_DISTRIBUTION}!g" \ -e "s!@REPOMGR_DEPLOY_URL@!${REPOMGR_DEPLOY_URL}!g" \ - -e "s!@REPOMGR_HOST@!${REPOMGR_HOST}!g" \ + -e "s!@REPOMGR_ORIGIN@!${REPOMGR_ORIGIN}!g" \ "$@" } @@ -369,14 +369,8 @@ else replace_vars "${SRC_DOCKER_DIR}/apt/stx.sources.list.in" >"${BUILDDIR}/apt/stx.sources.list" fi - # preferences: instantiate template once for every host in stx.sources.list - unique_hosts=$(\grep -v -E '^\s*(#.*)?$' "${BUILDDIR}/apt/stx.sources.list" | sed -n -r 's#.*(https?|ftp)://([^/:[:space:]]+).*#\2#p' | sort -u) - echo -n >"${BUILDDIR}/apt/stx.preferences" - for host in $unique_hosts ; do - REPOMGR_HOST="$host" replace_vars "${SRC_DOCKER_DIR}/apt/stx.preferences.part.in" >>"${BUILDDIR}/apt/stx.preferences" - echo >>"${BUILDDIR}/apt/stx.preferences" - done - unset host unique_hosts + # preferences: instantiate template with REPOMGR_ORIGIN from environment + replace_vars "${SRC_DOCKER_DIR}/apt/stx.preferences.part.in" >>"${BUILDDIR}/apt/stx.preferences" unset -f replace_vars fi diff --git a/build-tools/build-docker-images/stx-debian/Dockerfile.stable b/build-tools/build-docker-images/stx-debian/Dockerfile.stable index 5b0acfd7..aee50048 100644 --- a/build-tools/build-docker-images/stx-debian/Dockerfile.stable +++ b/build-tools/build-docker-images/stx-debian/Dockerfile.stable @@ -18,17 +18,6 @@ COPY apt/debian.sources.list /etc/apt/sources.list.d/debian.list.disabled COPY apt/stx.sources.list /etc/apt/sources.list.d/stx.list.disabled COPY apt/stx.preferences /etc/apt/preferences.d/stx -# Enable stx repo -RUN cp /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list - -# Clean apt cache -RUN apt-get clean && rm -rf /var/lib/apt/lists/* - -# Upgrade base packages to versions in the managed repos -RUN apt-get -y update && \ - apt-get -y upgrade && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - # repo templates: # /etc/apt/sources.list.d/ # debian.list.disabled - vanilla debian repos @@ -48,18 +37,64 @@ RUN apt-get -y update && \ # Enabling the upstream repos ("debian.list") is dangerous because it # may conflict with packages in stx.list. # +# +# FIXME: apt evaluates these files in alphabetical order, so stx.list +# comes after debian.list. When the local binary repo contains +# the same package/version as the debian repo, apt will download +# it from debian, regardless of the priority in /etc/apt/preferences. +# We should rename these files to make stx.list sort before +# debian.list. This would affect Loci scripts in +# loci/docker/stx-scripts/ +# +# +# Upgrade base packages to versions in managed repos +# +RUN cp -f /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list && \ + apt-get -y update && \ + apt-get -y upgrade && \ + rm -f /etc/apt/sources.list.d/stx.list && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# +# Install packages provided only by debian. +# FIXME: move these packages + their dependencies to debian download lists in +# starlingx/tools to avoid referencing the debian repo at all. +# +RUN cp -f /etc/apt/sources.list.d/debian.list.disabled /etc/apt/sources.list.d/debian.list && \ + cp -f /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list && \ + apt-get update -y && \ + apt-get install -y \ + libapache2-mod-wsgi-py3 \ + python3-setuptools \ + build-essential \ + && \ + rm -f /etc/apt/sources.list.d/debian.list && \ + rm -f /etc/apt/sources.list.d/stx.list && \ + apt-get clean && \ + rm -rf /var/lib/apt/files/* + +# +# Enable stx repo only. Packages installs below this point will use +# only the managed locally-built & 3rd-party repos. +# +RUN cp /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list + +# +# Install required packages +# RUN apt-get update -y && \ apt-get upgrade -y && \ apt-get install -y \ -# FIXME: uncomment once qemu is ported to debian (starlingx/integ) -# qemu-utils \ openssh-client \ python3 \ python3-pip \ python3-wheel \ - libapache2-mod-wsgi-py3 \ - ; +# FIXME: uncomment once qemu is ported to debian (starlingx/integ) +# qemu-utils \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/files/* # FIXME: these packages are not required by most docker images inheriting # from this image. However these Python modules are not buildable from @@ -69,11 +104,12 @@ RUN apt-get update -y && \ # # A better solution would be to omit them here, but install them in each # project that requires them; or add wheel subpackages to these DEBs. -RUN apt-get install -y \ - python3-thriftpy \ - python3-nss \ - python-nss - -# Delete apt cache -RUN apt-get clean && \ - rm -rf /var/lib/apt/lists/* +RUN apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install -y \ + python3-thriftpy \ + python3-nss \ + python-nss \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/files/* diff --git a/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in b/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in index 5863a2e0..367e3d99 100644 --- a/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in +++ b/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in @@ -1,4 +1,4 @@ -Explanation: Prefer StarlingX repos over vanilla Debian +Explanation: Prefer StarlingX repos over anything else Package: * -Pin: origin "@REPOMGR_HOST@" +Pin: release o=@REPOMGR_ORIGIN@ Pin-Priority: 999 diff --git a/build-tools/stx/aptly_deb_usage.py b/build-tools/stx/aptly_deb_usage.py index 80554cc1..5bb10fed 100755 --- a/build-tools/stx/aptly_deb_usage.py +++ b/build-tools/stx/aptly_deb_usage.py @@ -47,12 +47,16 @@ SIGN_PASSWD = 'starlingx' class Deb_aptly(): - def __init__(self, url, logger): + def __init__(self, url, origin, logger): '''The basic interface to manage aptly database. ''' self.logger = logger self.url = url self.aptly = Client(self.url) self.logger.info('Aptly connected, version: %s', self.aptly.misc.version) + if origin: + self.origin = origin.strip() or None + else: + self.origin = None # Create a remote mirror(make sure the name has specified prefix) # Input @@ -291,10 +295,12 @@ class Deb_aptly(): # Add 'source' to publish source packages, if no source packages, that is also harmless. extra_param['architectures'] = mirror.architectures.append('source') extra_param['distribution'] = mirror.distribution + extra_param['origin'] = None else: # Only support binary_amd64 and source packages extra_param['architectures'] = ['amd64', 'source'] extra_param['distribution'] = None + extra_param['origin'] = self.origin extra_param['source_kind'] = 'snapshot' extra_param['sources'] = [{'Name': name}] @@ -304,7 +310,8 @@ class Deb_aptly(): task = self.aptly.publish.publish(source_kind='snapshot', sources=extra_param['sources'], architectures=extra_param['architectures'], prefix=extra_param['prefix'], distribution=extra_param['distribution'], - sign_gpgkey=SIGN_KEY, sign_passphrase=SIGN_PASSWD) + sign_gpgkey=SIGN_KEY, sign_passphrase=SIGN_PASSWD, + origin=extra_param['origin']) # In corner cases, "publish" may spend more than 60 seconds, cause # "wait_for_task_by_id" timeout. To cover such cases, we "wait_for_task_by_id" # up to 3 times, to enlarge the whole timeout to 180 seconds. diff --git a/build-tools/stx/build-image b/build-tools/stx/build-image index 1d7192d3..92869d22 100755 --- a/build-tools/stx/build-image +++ b/build-tools/stx/build-image @@ -435,7 +435,8 @@ if __name__ == "__main__": rmg_logger = logging.getLogger('repo_manager') utils.set_logger(rmg_logger) repo_manager = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'), - '/tmp/', rmg_logger) + '/tmp/', os.environ.get('REPOMGR_ORIGIN'), + rmg_logger) repo_manager.upload_pkg(REPO_BUILD, None) repo_manager.upload_pkg(REPO_BINARY, None) diff --git a/build-tools/stx/build-pkgs b/build-tools/stx/build-pkgs index 059ad036..e78edfd2 100755 --- a/build-tools/stx/build-pkgs +++ b/build-tools/stx/build-pkgs @@ -35,6 +35,7 @@ import yaml BUILDER_URL = os.environ.get('BUILDER_URL') REPOMGR_URL = os.environ.get('REPOMGR_URL') +REPOMGR_ORIGIN = os.environ.get('REPOMGR_ORIGIN') BUILD_ROOT = os.environ.get('MY_BUILD_PKG_DIR') STX_ROOT = os.environ.get('MY_REPO_ROOT_DIR') PKGBUILDER_ROOT = "/localdisk/pkgbuilder" @@ -274,7 +275,8 @@ class BuildController(): rlogger = logging.getLogger('repo_manager') utils.set_logger(rlogger) self.kits['repo_mgr'] = repo_manage.RepoMgr('aptly', REPOMGR_URL, - '/tmp', rlogger) + '/tmp', REPOMGR_ORIGIN, + rlogger) logger.debug("Successful created repo manager") @property diff --git a/build-tools/stx/debdownloader b/build-tools/stx/debdownloader index 7a26b90a..6988cc73 100755 --- a/build-tools/stx/debdownloader +++ b/build-tools/stx/debdownloader @@ -87,7 +87,8 @@ if __name__ == "__main__": # Create local binary repo with repo manager repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'), - '/tmp/', logger) + '/tmp/', os.environ.get('REPOMGR_ORIGIN'), + logger) repomgr.upload_pkg(REPO_BIN, None) with open(sys.argv[1], 'r') as flist: diff --git a/build-tools/stx/downloader b/build-tools/stx/downloader index e5c4b570..1e309bf3 100755 --- a/build-tools/stx/downloader +++ b/build-tools/stx/downloader @@ -165,7 +165,8 @@ class BaseDownloader(): rlogger = logging.getLogger('repo_manager') utils.set_logger(rlogger) self.repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'), - '/tmp/', rlogger) + '/tmp/', os.environ.get('REPOMGR_ORIGIN'), + rlogger) def clean(self): if os.path.exists(self.dl_dir): diff --git a/build-tools/stx/repo_manage.py b/build-tools/stx/repo_manage.py index b22bec76..761cdc8b 100755 --- a/build-tools/stx/repo_manage.py +++ b/build-tools/stx/repo_manage.py @@ -33,6 +33,7 @@ import utils REPOMGR_URL = os.environ.get('REPOMGR_URL') +REPOMGR_ORIGIN = os.environ.get('REPOMGR_ORIGIN') REPOMGR_DEPLOY_URL = os.environ.get('REPOMGR_DEPLOY_URL') APTFETCH_JOBS = 10 @@ -218,9 +219,9 @@ class RepoMgr(): remote repo: mirror of another repository. shouldn't insert or remove packages from it.. local repo: a local repository, we can insert/remove packages into/from them. ''' - def __init__(self, repoType, repoURL, workdir, logger): + def __init__(self, repoType, repoURL, workdir, origin, logger): if repoType == 'aptly': - self.repo = aptly_deb_usage.Deb_aptly(repoURL, logger) + self.repo = aptly_deb_usage.Deb_aptly(repoURL, origin, logger) else: raise Exception('Currently, only aptly repository supported') @@ -662,44 +663,44 @@ utils.set_logger(applogger) def _handleDownload(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, REPOMGR_ORIGIN, applogger) kwargs = {'sources_list': args.sources_list, 'deb_list': args.deb_list, 'dsc_list': args.dsc_list} repomgr.download(args.name, **kwargs, no_clear=args.no_clear) def _handleSync(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, REPOMGR_ORIGIN, applogger) kwargs = {'sources_list': args.sources_list, 'deb_list': args.deb_list, 'dsc_list': args.dsc_list} repomgr.sync(args.name, args.repo_list, **kwargs, no_clear=args.no_clear) def _handleMirror(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) kwargs = {'url': args.url, 'distribution': args.distribution, 'component': args.component, 'architectures': args.architectures, 'with_sources': args.with_sources} repomgr.mirror(args.name, **kwargs) def _handleMerge(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.merge(args.name, args.repo_list) def _handleUploadPkg(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.upload_pkg(args.repository, args.package) def _handleDeletePkg(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.delete_pkg(args.repository, args.package_name, args.package_type, pkg_version=args.package_version) def _handleSearchPkg(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) if args.package_type == 'binary': repomgr.search_pkg(args.repository, args.package_name, pkg_version=args.package_version, binary=True) @@ -709,22 +710,22 @@ def _handleSearchPkg(args): def _handleRemoveRope(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.remove_repo(args.repository) def _handleList(_args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.list() def _handleListPkgs(args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.list_pkgs(args.repository) def _handleClean(_args): - repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger) repomgr.clean()