build-tools: apt repo priority based on "Origin"
* build-docker-images/stx-debian/stx.preferences.part.in
* build-docker-images/build-base-image.sh
This file sets base image priorities for apt repos that match certain
properties. Formerly we used higher priority for repos hosted by the
build server, as opposed to debian.org or others. This doesn't work
when using a Debian mirror hosted on the build server itself, since
the hostname of both repos are equal.
Solution: increase priority for repos whose "Release" file contains
the field "Origin: $REPOMGR_ORIGIN" and make sure aptly adds that
field to its Release file. The value comes from the environment and
should be set by the build container.
* stx/aptly_deb_usage.py:
Add an "Origin" field to non-mirror publications, value taken from
environment REPOMGR_ORIGIN
* build-docker-images/stx-debian/Dockerfile.stable
Improvements to package conflict resolution and docker FS
caching-related issues:
- Upgrade base packages to versions in managed repos before doing
anything else
- Install packages provided by upstream debian in a separate RUN
command/docker FS layer
- Make sure each "apt-get install" is in its own RUN command and is
preceded with "apt-get update" -- to avoid using stale metadata due
to "docker build" FS layer caching
TESTS
======================
- Define REPOMGR_ORIGIN in container environment
- Run downloader & build-pkgs & make sure generated repos' Release file
contains "Origin: starlingx"
- Build base image & make sure its apt.preferences contains the priority
rule for "Origin: starlingx"
Story: 2010055
Task: 45729
Change-Id: Ibaafbfbeef408904d216265168daa466d90fc7f2
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
This commit is contained in:
Davlet Panech
parent
e5e7d8be7f
commit
85ecb712be
@ -333,7 +333,7 @@ EOF
|
||||
else
|
||||
|
||||
# These env vars must be defined in debian builder pods
|
||||
for var in DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION REPOMGR_DEPLOY_URL ; do
|
||||
for var in DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION REPOMGR_DEPLOY_URL REPOMGR_ORIGIN ; do
|
||||
if [[ -z "${!var}" ]] ; then
|
||||
echo "$var must be defined in the environment!" >&2
|
||||
exit 1
|
||||
@ -347,7 +347,7 @@ else
|
||||
-e "s!@DEBIAN_SECURITY_SNAPSHOT@!${DEBIAN_SECURITY_SNAPSHOT}!g" \
|
||||
-e "s!@DEBIAN_DISTRIBUTION@!${DEBIAN_DISTRIBUTION}!g" \
|
||||
-e "s!@REPOMGR_DEPLOY_URL@!${REPOMGR_DEPLOY_URL}!g" \
|
||||
-e "s!@REPOMGR_HOST@!${REPOMGR_HOST}!g" \
|
||||
-e "s!@REPOMGR_ORIGIN@!${REPOMGR_ORIGIN}!g" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
@ -369,14 +369,8 @@ else
|
||||
replace_vars "${SRC_DOCKER_DIR}/apt/stx.sources.list.in" >"${BUILDDIR}/apt/stx.sources.list"
|
||||
fi
|
||||
|
||||
# preferences: instantiate template once for every host in stx.sources.list
|
||||
unique_hosts=$(\grep -v -E '^\s*(#.*)?$' "${BUILDDIR}/apt/stx.sources.list" | sed -n -r 's#.*(https?|ftp)://([^/:[:space:]]+).*#\2#p' | sort -u)
|
||||
echo -n >"${BUILDDIR}/apt/stx.preferences"
|
||||
for host in $unique_hosts ; do
|
||||
REPOMGR_HOST="$host" replace_vars "${SRC_DOCKER_DIR}/apt/stx.preferences.part.in" >>"${BUILDDIR}/apt/stx.preferences"
|
||||
echo >>"${BUILDDIR}/apt/stx.preferences"
|
||||
done
|
||||
unset host unique_hosts
|
||||
# preferences: instantiate template with REPOMGR_ORIGIN from environment
|
||||
replace_vars "${SRC_DOCKER_DIR}/apt/stx.preferences.part.in" >>"${BUILDDIR}/apt/stx.preferences"
|
||||
unset -f replace_vars
|
||||
fi
|
||||
|
||||
|
||||
@ -18,17 +18,6 @@ COPY apt/debian.sources.list /etc/apt/sources.list.d/debian.list.disabled
|
||||
COPY apt/stx.sources.list /etc/apt/sources.list.d/stx.list.disabled
|
||||
COPY apt/stx.preferences /etc/apt/preferences.d/stx
|
||||
|
||||
# Enable stx repo
|
||||
RUN cp /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list
|
||||
|
||||
# Clean apt cache
|
||||
RUN apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Upgrade base packages to versions in the managed repos
|
||||
RUN apt-get -y update && \
|
||||
apt-get -y upgrade && \
|
||||
apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# repo templates:
|
||||
# /etc/apt/sources.list.d/
|
||||
# debian.list.disabled - vanilla debian repos
|
||||
@ -48,18 +37,64 @@ RUN apt-get -y update && \
|
||||
# Enabling the upstream repos ("debian.list") is dangerous because it
|
||||
# may conflict with packages in stx.list.
|
||||
#
|
||||
#
|
||||
# FIXME: apt evaluates these files in alphabetical order, so stx.list
|
||||
# comes after debian.list. When the local binary repo contains
|
||||
# the same package/version as the debian repo, apt will download
|
||||
# it from debian, regardless of the priority in /etc/apt/preferences.
|
||||
# We should rename these files to make stx.list sort before
|
||||
# debian.list. This would affect Loci scripts in
|
||||
# loci/docker/stx-scripts/
|
||||
#
|
||||
|
||||
#
|
||||
# Upgrade base packages to versions in managed repos
|
||||
#
|
||||
RUN cp -f /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list && \
|
||||
apt-get -y update && \
|
||||
apt-get -y upgrade && \
|
||||
rm -f /etc/apt/sources.list.d/stx.list && \
|
||||
apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
#
|
||||
# Install packages provided only by debian.
|
||||
# FIXME: move these packages + their dependencies to debian download lists in
|
||||
# starlingx/tools to avoid referencing the debian repo at all.
|
||||
#
|
||||
RUN cp -f /etc/apt/sources.list.d/debian.list.disabled /etc/apt/sources.list.d/debian.list && \
|
||||
cp -f /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list && \
|
||||
apt-get update -y && \
|
||||
apt-get install -y \
|
||||
libapache2-mod-wsgi-py3 \
|
||||
python3-setuptools \
|
||||
build-essential \
|
||||
&& \
|
||||
rm -f /etc/apt/sources.list.d/debian.list && \
|
||||
rm -f /etc/apt/sources.list.d/stx.list && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/files/*
|
||||
|
||||
#
|
||||
# Enable stx repo only. Packages installs below this point will use
|
||||
# only the managed locally-built & 3rd-party repos.
|
||||
#
|
||||
RUN cp /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list
|
||||
|
||||
#
|
||||
# Install required packages
|
||||
#
|
||||
RUN apt-get update -y && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y \
|
||||
# FIXME: uncomment once qemu is ported to debian (starlingx/integ)
|
||||
# qemu-utils \
|
||||
openssh-client \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
libapache2-mod-wsgi-py3 \
|
||||
;
|
||||
# FIXME: uncomment once qemu is ported to debian (starlingx/integ)
|
||||
# qemu-utils \
|
||||
&& \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/files/*
|
||||
|
||||
# FIXME: these packages are not required by most docker images inheriting
|
||||
# from this image. However these Python modules are not buildable from
|
||||
@ -69,11 +104,12 @@ RUN apt-get update -y && \
|
||||
#
|
||||
# A better solution would be to omit them here, but install them in each
|
||||
# project that requires them; or add wheel subpackages to these DEBs.
|
||||
RUN apt-get install -y \
|
||||
python3-thriftpy \
|
||||
python3-nss \
|
||||
python-nss
|
||||
|
||||
# Delete apt cache
|
||||
RUN apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
RUN apt-get update -y && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y \
|
||||
python3-thriftpy \
|
||||
python3-nss \
|
||||
python-nss \
|
||||
&& \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/files/*
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
Explanation: Prefer StarlingX repos over vanilla Debian
|
||||
Explanation: Prefer StarlingX repos over anything else
|
||||
Package: *
|
||||
Pin: origin "@REPOMGR_HOST@"
|
||||
Pin: release o=@REPOMGR_ORIGIN@
|
||||
Pin-Priority: 999
|
||||
|
||||
@ -47,12 +47,16 @@ SIGN_PASSWD = 'starlingx'
|
||||
|
||||
|
||||
class Deb_aptly():
|
||||
def __init__(self, url, logger):
|
||||
def __init__(self, url, origin, logger):
|
||||
'''The basic interface to manage aptly database. '''
|
||||
self.logger = logger
|
||||
self.url = url
|
||||
self.aptly = Client(self.url)
|
||||
self.logger.info('Aptly connected, version: %s', self.aptly.misc.version)
|
||||
if origin:
|
||||
self.origin = origin.strip() or None
|
||||
else:
|
||||
self.origin = None
|
||||
|
||||
# Create a remote mirror(make sure the name has specified prefix)
|
||||
# Input
|
||||
@ -291,10 +295,12 @@ class Deb_aptly():
|
||||
# Add 'source' to publish source packages, if no source packages, that is also harmless.
|
||||
extra_param['architectures'] = mirror.architectures.append('source')
|
||||
extra_param['distribution'] = mirror.distribution
|
||||
extra_param['origin'] = None
|
||||
else:
|
||||
# Only support binary_amd64 and source packages
|
||||
extra_param['architectures'] = ['amd64', 'source']
|
||||
extra_param['distribution'] = None
|
||||
extra_param['origin'] = self.origin
|
||||
|
||||
extra_param['source_kind'] = 'snapshot'
|
||||
extra_param['sources'] = [{'Name': name}]
|
||||
@ -304,7 +310,8 @@ class Deb_aptly():
|
||||
task = self.aptly.publish.publish(source_kind='snapshot', sources=extra_param['sources'],
|
||||
architectures=extra_param['architectures'], prefix=extra_param['prefix'],
|
||||
distribution=extra_param['distribution'],
|
||||
sign_gpgkey=SIGN_KEY, sign_passphrase=SIGN_PASSWD)
|
||||
sign_gpgkey=SIGN_KEY, sign_passphrase=SIGN_PASSWD,
|
||||
origin=extra_param['origin'])
|
||||
# In corner cases, "publish" may spend more than 60 seconds, cause
|
||||
# "wait_for_task_by_id" timeout. To cover such cases, we "wait_for_task_by_id"
|
||||
# up to 3 times, to enlarge the whole timeout to 180 seconds.
|
||||
|
||||
@ -435,7 +435,8 @@ if __name__ == "__main__":
|
||||
rmg_logger = logging.getLogger('repo_manager')
|
||||
utils.set_logger(rmg_logger)
|
||||
repo_manager = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'),
|
||||
'/tmp/', rmg_logger)
|
||||
'/tmp/', os.environ.get('REPOMGR_ORIGIN'),
|
||||
rmg_logger)
|
||||
repo_manager.upload_pkg(REPO_BUILD, None)
|
||||
repo_manager.upload_pkg(REPO_BINARY, None)
|
||||
|
||||
|
||||
@ -35,6 +35,7 @@ import yaml
|
||||
|
||||
BUILDER_URL = os.environ.get('BUILDER_URL')
|
||||
REPOMGR_URL = os.environ.get('REPOMGR_URL')
|
||||
REPOMGR_ORIGIN = os.environ.get('REPOMGR_ORIGIN')
|
||||
BUILD_ROOT = os.environ.get('MY_BUILD_PKG_DIR')
|
||||
STX_ROOT = os.environ.get('MY_REPO_ROOT_DIR')
|
||||
PKGBUILDER_ROOT = "/localdisk/pkgbuilder"
|
||||
@ -274,7 +275,8 @@ class BuildController():
|
||||
rlogger = logging.getLogger('repo_manager')
|
||||
utils.set_logger(rlogger)
|
||||
self.kits['repo_mgr'] = repo_manage.RepoMgr('aptly', REPOMGR_URL,
|
||||
'/tmp', rlogger)
|
||||
'/tmp', REPOMGR_ORIGIN,
|
||||
rlogger)
|
||||
logger.debug("Successful created repo manager")
|
||||
|
||||
@property
|
||||
|
||||
@ -87,7 +87,8 @@ if __name__ == "__main__":
|
||||
|
||||
# Create local binary repo with repo manager
|
||||
repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'),
|
||||
'/tmp/', logger)
|
||||
'/tmp/', os.environ.get('REPOMGR_ORIGIN'),
|
||||
logger)
|
||||
repomgr.upload_pkg(REPO_BIN, None)
|
||||
|
||||
with open(sys.argv[1], 'r') as flist:
|
||||
|
||||
@ -165,7 +165,8 @@ class BaseDownloader():
|
||||
rlogger = logging.getLogger('repo_manager')
|
||||
utils.set_logger(rlogger)
|
||||
self.repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'),
|
||||
'/tmp/', rlogger)
|
||||
'/tmp/', os.environ.get('REPOMGR_ORIGIN'),
|
||||
rlogger)
|
||||
|
||||
def clean(self):
|
||||
if os.path.exists(self.dl_dir):
|
||||
|
||||
@ -33,6 +33,7 @@ import utils
|
||||
|
||||
|
||||
REPOMGR_URL = os.environ.get('REPOMGR_URL')
|
||||
REPOMGR_ORIGIN = os.environ.get('REPOMGR_ORIGIN')
|
||||
REPOMGR_DEPLOY_URL = os.environ.get('REPOMGR_DEPLOY_URL')
|
||||
|
||||
APTFETCH_JOBS = 10
|
||||
@ -218,9 +219,9 @@ class RepoMgr():
|
||||
remote repo: mirror of another repository. shouldn't insert or remove packages from it..
|
||||
local repo: a local repository, we can insert/remove packages into/from them.
|
||||
'''
|
||||
def __init__(self, repoType, repoURL, workdir, logger):
|
||||
def __init__(self, repoType, repoURL, workdir, origin, logger):
|
||||
if repoType == 'aptly':
|
||||
self.repo = aptly_deb_usage.Deb_aptly(repoURL, logger)
|
||||
self.repo = aptly_deb_usage.Deb_aptly(repoURL, origin, logger)
|
||||
else:
|
||||
raise Exception('Currently, only aptly repository supported')
|
||||
|
||||
@ -662,44 +663,44 @@ utils.set_logger(applogger)
|
||||
|
||||
|
||||
def _handleDownload(args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, REPOMGR_ORIGIN, applogger)
|
||||
kwargs = {'sources_list': args.sources_list, 'deb_list': args.deb_list,
|
||||
'dsc_list': args.dsc_list}
|
||||
repomgr.download(args.name, **kwargs, no_clear=args.no_clear)
|
||||
|
||||
|
||||
def _handleSync(args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, args.basedir, REPOMGR_ORIGIN, applogger)
|
||||
kwargs = {'sources_list': args.sources_list, 'deb_list': args.deb_list,
|
||||
'dsc_list': args.dsc_list}
|
||||
repomgr.sync(args.name, args.repo_list, **kwargs, no_clear=args.no_clear)
|
||||
|
||||
|
||||
def _handleMirror(args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger)
|
||||
kwargs = {'url': args.url, 'distribution': args.distribution, 'component': args.component,
|
||||
'architectures': args.architectures, 'with_sources': args.with_sources}
|
||||
repomgr.mirror(args.name, **kwargs)
|
||||
|
||||
|
||||
def _handleMerge(args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger)
|
||||
repomgr.merge(args.name, args.repo_list)
|
||||
|
||||
|
||||
def _handleUploadPkg(args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger)
|
||||
repomgr.upload_pkg(args.repository, args.package)
|
||||
|
||||
|
||||
def _handleDeletePkg(args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger)
|
||||
repomgr.delete_pkg(args.repository, args.package_name, args.package_type,
|
||||
pkg_version=args.package_version)
|
||||
|
||||
|
||||
def _handleSearchPkg(args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger)
|
||||
if args.package_type == 'binary':
|
||||
repomgr.search_pkg(args.repository, args.package_name, pkg_version=args.package_version,
|
||||
binary=True)
|
||||
@ -709,22 +710,22 @@ def _handleSearchPkg(args):
|
||||
|
||||
|
||||
def _handleRemoveRope(args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger)
|
||||
repomgr.remove_repo(args.repository)
|
||||
|
||||
|
||||
def _handleList(_args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger)
|
||||
repomgr.list()
|
||||
|
||||
|
||||
def _handleListPkgs(args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger)
|
||||
repomgr.list_pkgs(args.repository)
|
||||
|
||||
|
||||
def _handleClean(_args):
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
|
||||
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', REPOMGR_ORIGIN, applogger)
|
||||
repomgr.clean()
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user