Add support for CentOS 8 Stream

This patch adds support for CentOS 8 Stream [1] to the centos-minimal
element. Users should set DIB_RELEASE=8-stream.

[1] https://www.centos.org/stream/

Change-Id: Id0825de735ab957c10daf35fb3c641f850cc6847

.zuul.d/jobs.yaml

@@ -23,6 +23,8 @@
         - apt-sources/test-sources
         - centos-minimal/7-build-succeeds
         - centos-minimal/8-build-succeeds
+        # TODO: enable when CentOS 8 Stream repos mirrored.
+        # - centos-minimal/8-stream-build-succeeds
         - fedora/build-succeeds
         - ubuntu-minimal/xenial-build-succeeds
         - ubuntu-minimal/bionic-build-succeeds
@@ -161,6 +163,19 @@
         release: '8'
         mirror: "http://{{ zuul_site_mirror_fqdn }}/centos"
 
+- job:
+    name: dib-nodepool-functional-openstack-centos-8-stream-src
+    description: |
+      Test building and booting a Centos 8 Stream image with Nodepool and
+      OpenStack.
+    parent: dib-nodepool-functional-src-base
+    vars:
+      nodepool_diskimage:
+        base_element: centos-minimal
+        release: '8-stream'
+        # TODO: set when CentOS 8 Stream repos mirrored.
+        # mirror: "http://{{ zuul_site_mirror_fqdn }}/centos"
+
 - job:
     name: dib-nodepool-functional-openstack-fedora-30-src
     description: |

.zuul.d/project.yaml

@@ -16,6 +16,7 @@
         - nodepool-build-image-siblings
         - dib-nodepool-functional-openstack-centos-7-src
         - dib-nodepool-functional-openstack-centos-8-src
+        - dib-nodepool-functional-openstack-centos-8-stream-src
         - dib-nodepool-functional-openstack-fedora-31-src
         - dib-nodepool-functional-openstack-ubuntu-xenial-src
         - dib-nodepool-functional-openstack-ubuntu-bionic-src
@@ -46,6 +47,7 @@
         - nodepool-build-image-siblings
         - dib-nodepool-functional-openstack-centos-7-src
         - dib-nodepool-functional-openstack-centos-8-src
+        - dib-nodepool-functional-openstack-centos-8-stream-src
         - dib-nodepool-functional-openstack-fedora-31-src
         - dib-nodepool-functional-openstack-ubuntu-xenial-src
         - dib-nodepool-functional-openstack-ubuntu-bionic-src

diskimage_builder/elements/centos-minimal/README.rst

@@ -6,8 +6,8 @@ Create a minimal image based on CentOS
 Use of this element will require 'yum' and 'yum-utils' to be installed on
 Ubuntu and Debian. Nothing additional is needed on Fedora or CentOS.
 
-By default this builds CentOS 7 images.  Set ``DIB_RELEASE`` to ``7``
+By default this builds CentOS 7 images.  Set ``DIB_RELEASE`` to ``7``,
-or ``8`` to explicitly select the release.
+``8`` or ``8-stream`` to explicitly select the release.
 
 For CentOS 7, by default, ``DIB_YUM_MINIMAL_CREATE_INTERFACES`` is set
 to enable the creation of
@@ -15,5 +15,5 @@ to enable the creation of
 DHCP on the ``eth0`` & ``eth1`` interfaces.  If you do not have these
 interfaces, or if you are using something else to setup the network
 such as cloud-init, glean or network-manager, you would want to set
-this to ``0``.  For CentOS 8, this is set to ``0`` by default as the
+this to ``0``.  For CentOS 8 and CentOS 8 Stream, this is set to ``0`` by
-system uses NetworkManager by default.
+default as the system uses NetworkManager by default.

diskimage_builder/elements/centos-minimal/environment.d/10-centos-distro-name.bash

@@ -4,7 +4,7 @@ export DIB_RELEASE=${DIB_RELEASE:-7}
 # by default, enable DHCP configuration of eth0 & eth1 in network
 # scripts for centos 7.  See yum-minimal for full details.  CentOS 8
 # does not come with network-scripts by default so avoid this there.
-if [[ ${DIB_RELEASE} -le 7 ]]; then
+if [[ "${DIB_RELEASE}" < "8" ]]; then
     export DIB_YUM_MINIMAL_CREATE_INTERFACES=${DIB_YUM_MINIMAL_CREATE_INTERFACES:-1}
 else
     export DIB_YUM_MINIMAL_CREATE_INTERFACES=${DIB_YUM_MINIMAL_CREATE_INTERFACES:-0}

diskimage_builder/elements/centos-minimal/environment.d/11-yum-dnf.bash

@@ -1,5 +1,5 @@
-if [ $DIB_RELEASE -ge 8 ]; then
+if [[ $DIB_RELEASE == "7" ]]; then
-    export YUM=dnf
-else
     export YUM=yum
+else
+    export YUM=dnf
 fi

diskimage_builder/elements/centos-minimal/test-elements/8-stream-build-succeeds/README.rst

@@ -0,0 +1 @@
+Verify we can build a centos-minimal image.

diskimage_builder/elements/centos-minimal/test-elements/8-stream-build-succeeds/element-deps

@@ -0,0 +1 @@
+openstack-ci-mirrors

diskimage_builder/elements/centos-minimal/test-elements/8-stream-build-succeeds/environment.d/09-set-distro.bash

@@ -0,0 +1 @@
+export DIB_RELEASE='8-stream'

diskimage_builder/elements/centos-minimal/yum.repos.d/8-stream/appstream.repo

@@ -0,0 +1,5 @@
+[Stream-AppStream]
+name=CentOS-Stream - AppStream
+mirrorlist=http://mirrorlist.centos.org/?release=8-stream&arch=$basearch&repo=AppStream
+gpgcheck=0
+

diskimage_builder/elements/centos-minimal/yum.repos.d/8-stream/base.repo

@@ -0,0 +1,5 @@
+[Stream-BaseOS]
+name=CentOS-Stream - Base
+mirrorlist=http://mirrorlist.centos.org/?release=8-stream&arch=$basearch&repo=BaseOS
+gpgcheck=0
+

diskimage_builder/elements/dib-python/environment.d/50-dib-python-version

@@ -16,9 +16,9 @@ if [ -z "${DIB_PYTHON_VERSION:-}" ]; then
         # TODO(nmagnezi): Remove this when the 'rhel7' element gets replaced by 'rhel'
         DIB_PYTHON_VERSION=2
     elif [[ "$DISTRO_NAME" =~ (rhel|centos) ]]; then
-        if [ "$DIB_RELEASE" -le 7 ]; then
+        if [[ "${DIB_RELEASE}" == "7" ]]; then
             DIB_PYTHON_VERSION=2
-        elif [ "$DIB_RELEASE" -ge 8 ]; then
+        else
             DIB_PYTHON_VERSION=3
         fi
     elif [ "$DISTRO_NAME" == "opensuse" ]; then

diskimage_builder/elements/dib-python/pre-install.d/01-dib-python

@@ -6,7 +6,7 @@ fi
 set -eu
 set -o pipefail
 
-if [[ ${DISTRO_NAME} =~ (centos|rhel) && ${DIB_RELEASE} == 8 ]]; then
+if [[ ${DISTRO_NAME} =~ (centos|rhel) && ${DIB_RELEASE} > 7 ]]; then
     # RHEL8 has a system python, separate from the user python.  What
     # a good idea, abstracting the python binary for system scripts!
     # :)  Use it for dib-python.

diskimage_builder/elements/pip-and-virtualenv/environment.d/51-pip-and-virutalenv-default.bash

@@ -2,7 +2,7 @@
 # upgrading packaged system pip/setuptools/virtualenv binaries with
 # upstream non-packaged versions, we wish to avoid this completely on
 # modern distros.
-if [[ $DISTRO_NAME =~ (centos|rhel) && $DIB_RELEASE -ge 8 ]]; then
+if [[ $DISTRO_NAME =~ (centos|rhel) && $DIB_RELEASE > 7 ]]; then
     export DIB_INSTALLTYPE_pip_and_virtualenv=${DIB_INSTALLTYPE_pip_and_virtualenv:-package}
 
     if [[ ${DIB_INSTALLTYPE_pip_and_virtualenv} == "source" ]]; then

diskimage_builder/elements/pkg-map/bin/pkg-map

@@ -38,6 +38,12 @@ def os_family(distro):
     return family
 
 
+def os_major_release(release):
+    if release in ['8-stream']:
+        return '8'
+    return release
+
+
 def main():
     parser = argparse.ArgumentParser(
         description="Translate package name to distro specific name."
@@ -164,7 +170,8 @@ def main():
     if 'release' in package_names:
         try:
             # release is a sub-concept of distro
-            release_map = package_names['release'][args.distro][args.release]
+            release = os_major_release(args.release)
+            release_map = package_names['release'][args.distro][release]
             name_map.update(release_map)
         except KeyError:
             pass

diskimage_builder/elements/simple-init/environment.d/15-simple-init-networkmanager

@@ -4,7 +4,7 @@ else
     export DIB_SIMPLE_INIT_NETWORKMANAGER=${DIB_SIMPLE_INIT_NETWORKMANAGER:-0}
 fi
 
-if [[ ${DISTRO_NAME} == "centos" && $DIB_RELEASE -ge 8 ]] || [[ ${DISTRO_NAME} == "fedora" ]]; then
+if [[ ${DISTRO_NAME} == "centos" && $DIB_RELEASE > 7 ]] || [[ ${DISTRO_NAME} == "fedora" ]]; then
     if [[ $DIB_SIMPLE_INIT_NETWORKMANAGER == 0 ]]; then
         echo "simple-init only supports NetworkManager mode on Fedora and CentOS 8"
         exit 1

diskimage_builder/elements/simple-init/post-install.d/80-simple-init

@@ -12,15 +12,13 @@ case "$DIB_INIT_SYSTEM" in
         exit 0
         ;;
     systemd)
-        if [[ ${DISTRO_NAME} == centos && ${DIB_RELEASE} -eq 7 ]]; then
+        if [[ ${DISTRO_NAME} == centos && ${DIB_RELEASE} == 7 ]]; then
             if [[ ${DIB_SIMPLE_INIT_NETWORKMANAGER} != 1 ]]; then
                 # NOTE(pabelanger): Glean requires network.service for
                 # these platforms when not using networkmanager
-                if [[ ${DIB_RELEASE} -lt 8 ]]; then
+                # Not available on Centos 8
-                    # Not available on Centos 8
+                # NOTE(ianw) we should evaluate if Fedora needs this
-                    # NOTE(ianw) we should evaluate if Fedora needs this
+                systemctl enable network.service
-                    systemctl enable network.service
-                fi
             fi
         elif [[ ${DISTRO_NAME} =~ (opensuse) ]]; then
             # on suse, this is named wicked.service, but it's the same

diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot

@@ -90,8 +90,11 @@ function _install_repos {
     fi
 
     # CentOS 8.1 split repositories and GPG keys out into subpackages
-    if [[ ${DISTRO_NAME} = centos && ${DIB_RELEASE} -ge 8 ]]; then
+    if [[ ${DISTRO_NAME} = centos && ${DIB_RELEASE} > "7" ]]; then
         packages+="centos-repos centos-gpg-keys "
+        if [[ "$DIB_RELEASE" =~ (stream) ]]; then
+            packages+="centos-release-stream "
+        fi
     fi
 
     # By default, parent elements (fedora-minimal, centos-minimal)
@@ -117,7 +120,7 @@ function _install_repos {
     local temp_tmp
     temp_tmp=$(mktemp -d)
     TMPDIR=${temp_tmp} yumdownloader --verbose \
-        --releasever=$DIB_RELEASE \
+        --releasever=${DIB_RELEASE/-*/} \
         --setopt=reposdir=$repo \
         --setopt=cachedir=$temp_tmp \
         --destdir=$WORKING \
@@ -190,7 +193,7 @@ function _install_pkg_manager {
         local _extra_pkgs=""
 
         if [[ $DISTRO_NAME == "fedora" ]] || \
-            [[ $DISTRO_NAME == "centos" && $DIB_RELEASE -ge 8 ]]; then
+            [[ $DISTRO_NAME == "centos" && $DIB_RELEASE > "7" ]]; then
             # glibc from F24 onwards has split locales into "langpack"
             # packages.  Host yum doesn't understand the
             # weak-dependencies glibc now uses to get the
@@ -209,14 +212,25 @@ function _install_pkg_manager {
         # Really all we can do is pre-install the right thing
         _extra_pkgs+="coreutils "
 
+        # Legacy yum reads vars from directory /etc/yum/vars and, unlike dnf,
+        # does not provide setopt=varsdir. So, if $YUM is legacy yum and our
+        # target root is dnf, symlink dnf vars.
+        if [[ ! -d $TARGET_ROOT/etc/yum/vars ]]; then
+            sudo mkdir -p $TARGET_ROOT/etc/yum
+            sudo ln -s $TARGET_ROOT/etc/dnf/vars $TARGET_ROOT/etc/yum/vars
+        fi
+
         sudo -E yum -y \
             --disableexcludes=all \
             --setopt=cachedir=$YUM_CACHE/$ARCH/$DIB_RELEASE \
             --setopt=reposdir=$TARGET_ROOT/etc/yum.repos.d \
-            --releasever=$DIB_RELEASE \
+            --releasever=${DIB_RELEASE/-*/} \
             --installroot $TARGET_ROOT \
             install $@ ${_lang_pack} ${_extra_pkgs} && rc=$? || rc=$?
 
+        # we may have symlinked yum/vars -> dnf/vars, unset if so
+        sudo unset $TARGET_ROOT/etc/yum/vars 2>/dev/null || true
+
         # Note we've modified the base system's .rpmmacros.  Ensure we
         # clean it up *always*
         # sed makes it easy to remove last line, but not last n lines...
@@ -276,7 +290,7 @@ _install_repos
 # seemingly mismatched libraries.
 if [[ ${DISTRO_NAME} == 'fedora' ]]; then
     _install_pkg_manager dnf dnf-plugins-core curl
-elif [[ ${DISTRO_NAME} == centos && $DIB_RELEASE -ge 8 ]]; then
+elif [[ ${DISTRO_NAME} == centos && $DIB_RELEASE > "7" ]]; then
     _install_pkg_manager dnf dnf-plugins-core curl libcurl
 else
     _install_pkg_manager yum

releasenotes/notes/centos-minimal-8-stream-97021a2c92463dde.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    The ``centos-minimal`` supports CentOS 8 Stream builds when run with
+    ``DIB_RELEASE=8-stream``