Build CentOS Stream 8 images

This is part of the effort to allow a migration path
for operators using CentOS 7 and Train today.

Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/831265
Change-Id: I75373eb21c4269c20f72603ad4cf162a472de25b
(cherry picked from commit 67d6d28b62)

.zuul.d/base.yaml

@@ -31,6 +31,12 @@
       - name: primary
         label: centos-7
 
+- nodeset:
+    name: kolla-centos8-stream
+    nodes:
+      - name: primary
+        label: centos-8-stream
+
 - nodeset:
     name: kolla-ubuntu-bionic
     nodes:

.zuul.d/centos.yaml

@@ -4,8 +4,10 @@
       jobs:
         - kolla-build-centos-binary
         - kolla-build-centos-source
+        - kolla-build-centos8s-source
         - kolla-ansible-centos-source
         - kolla-ansible-centos-binary
+        - kolla-ansible-centos8s-source
         - tripleo-build-containers-centos-7:
             voting: true
             files:
@@ -18,7 +20,9 @@
       queue: kolla
       jobs:
         - kolla-build-centos-source
+        - kolla-build-centos8s-source
         - kolla-ansible-centos-source
+        - kolla-ansible-centos8s-source
         - tripleo-build-containers-centos-7:
             voting: true
             files:
@@ -31,10 +35,12 @@
       jobs:
         - kolla-publish-centos-source
         - kolla-publish-centos-binary
+        - kolla-publish-centos8s-source
     release:
       jobs:
         - kolla-publish-centos-source
         - kolla-publish-centos-binary
+        - kolla-publish-centos8s-source
     experimental:
       jobs:
         - kolla-ansible-bifrost-centos-source:
@@ -77,6 +83,12 @@
       base_distro: centos
       install_type: source
 
+- job:
+    name: kolla-build-centos8s-source
+    parent: kolla-build-centos-source
+    vars:
+      base_tag: stream8
+
 - job:
     name: kolla-publish-centos-source
     parent: kolla-build-centos-source
@@ -85,3 +97,9 @@
       publisher: true
     secrets:
       - kolla_dockerhub_creds
+
+- job:
+    name: kolla-publish-centos8s-source
+    parent: kolla-publish-centos-source
+    vars:
+      base_tag: stream8

doc/source/matrix_x86.csv

@@ -1,105 +1,105 @@
 Image,CentOS 7,,CentOS 8,,Ubuntu,,Debian
 ,Binary,Source,Binary,Source,Binary,Source,Binary,Source
 almanach (deprecated),N,C,N,N,N,C,N,C
-aodh,C,C,C,C,C,C,N,C
-barbican,C,T,C,T,C,C,N,C
+aodh,N,C,C,C,C,C,N,C
+barbican,N,T,C,T,C,C,N,C
 bifrost,N,T,N,T,N,C,N,N
 blazar,N,C,N,C,N,C,N,C
-ceilometer,C,C,C,C,C,C,N,C
-ceph,C,T,N,N,C,T,N,C
-certmonger,C,C,C,C,C,C,N,C
-chrony,T,T,T,T,T,T,N,T
-cinder,C,T,C,T,C,T,N,C
-cloudkitty,C,C,C,C,N,C,N,C
-collectd,C,C,C,C,C,C,N,C
-congress,C,C,C,C,N,C,N,C
-cron,T,T,T,T,T,T,N,T
+ceilometer,N,C,C,C,C,C,N,C
+ceph,N,T,N,N,C,T,N,C
+certmonger,N,C,C,C,C,C,N,C
+chrony,N,T,T,T,T,T,N,T
+cinder,N,T,C,T,C,T,N,C
+cloudkitty,N,C,C,C,N,C,N,C
+collectd,N,C,C,C,C,C,N,C
+congress,N,C,C,C,N,C,N,C
+cron,N,T,T,T,T,T,N,T
 cyborg,N,C,N,C,N,N,N,N
 cyborg-agent,N,C,N,N,N,N,N,N
-designate,C,C,C,C,C,C,N,C
-dind (deprecated),C,C,N,N,C,C,N,C
-dnsmasq,T,T,T,T,C,T,N,C
+designate,N,C,C,C,C,C,N,C
+dind (deprecated),N,C,N,N,C,C,N,C
+dnsmasq,N,T,T,T,C,T,N,C
 dragonflow (deprecated),N,C,N,N,N,C,N,C
-ec2-api,C,C,C,C,N,C,N,C
-elasticsearch (5),C,C,N,N,C,C,N,C
-elasticsearch (6),C,C,C,C,N,N,N,N
-etcd,C,T,C,T,C,T,N,C
-fluentd,T,T,T,T,T,T,N,T
+ec2-api,N,C,C,C,N,C,N,C
+elasticsearch (5),N,C,N,N,C,C,N,C
+elasticsearch (6),N,C,C,C,N,N,N,N
+etcd,N,T,C,T,C,T,N,C
+fluentd,N,T,T,T,T,T,N,T
 freezer,N,C,N,C,N,C,N,C
-glance,T,T,T,T,T,T,N,T
-gnocchi,C,C,C,C,C,C,N,C
-grafana,C,C,C,C,C,C,N,C
-hacluster,C,C,C,C,C,C,N,C
-hacluster-pcs,C,C,N,N,C,C,N,C
-haproxy,T,T,T,T,T,T,N,C
-heat,T,T,T,T,N,T,N,T
+glance,N,T,T,T,T,T,N,T
+gnocchi,N,C,C,C,C,C,N,C
+grafana,N,C,C,C,C,C,N,C
+hacluster,N,C,C,C,C,C,N,C
+hacluster-pcs,N,C,N,N,C,C,N,C
+haproxy,N,T,T,T,T,T,N,C
+heat,N,T,T,T,N,T,N,T
 helm-repository (deprecated),N,N,N,N,N,N,N,N
-horizon,T,T,T,T,T,T,N,T
-influxdb,C,C,C,C,C,C,N,C
-ironic,T,T,T,T,C,T,N,C
-ironic-neutron-agent,T,T,T,T,N,T,N,C
-ironic-inspector,T,T,T,T,C,T,N,C
-iscsid,T,T,T,T,T,T,N,C
-kafka,C,C,C,C,C,C,N,C
+horizon,N,T,T,T,T,T,N,T
+influxdb,N,C,C,C,C,C,N,C
+ironic,N,T,T,T,C,T,N,C
+ironic-neutron-agent,N,T,T,T,N,T,N,C
+ironic-inspector,N,T,T,T,C,T,N,C
+iscsid,N,T,T,T,T,T,N,C
+kafka,N,C,C,C,C,C,N,C
 karbor,N,C,N,C,N,C,N,C
-keepalived,T,T,T,T,T,T,N,C
-keystone,T,T,T,T,T,T,N,T
-kibana (5),C,C,N,N,C,C,N,C
-kibana (6),C,C,C,C,N,N,N,N
-kolla-toolbox,T,T,T,T,T,T,N,T
-kube (deprecated),C,C,N,N,C,C,N,C
-kubernetes-entrypoint (deprecated),C,C,N,N,C,C,N,C
-kubetoolbox (deprecated),C,C,N,N,C,C,N,C
+keepalived,N,T,T,T,T,T,N,C
+keystone,N,T,T,T,T,T,N,T
+kibana (5),N,C,N,N,C,C,N,C
+kibana (6),N,C,C,C,N,N,N,N
+kolla-toolbox,N,T,T,T,T,T,N,T
+kube (deprecated),N,C,N,N,C,C,N,C
+kubernetes-entrypoint (deprecated),N,C,N,N,C,C,N,C
+kubetoolbox (deprecated),N,C,N,N,C,C,N,C
 kuryr,N,T,N,T,N,T,N,C
-logstash (2),C,C,C,C,C,C,N,C
-logstash (6),C,C,C,C,N,N,N,N
-magnum,C,C,C,C,C,C,N,C
-manila,C,C,C,C,C,C,N,C
-mariadb,T,T,T,T,T,T,N,T
+logstash (2),N,C,C,C,C,C,N,C
+logstash (6),N,C,C,C,N,N,N,N
+magnum,N,C,C,C,C,C,N,C
+manila,N,C,C,C,C,C,N,C
+mariadb,N,T,T,T,T,T,N,T
 masakari,N,T,N,T,N,T,N,C
-memcached,T,T,T,T,T,T,N,C
-mistral,C,T,C,T,N,C,N,C
+memcached,N,T,T,T,T,T,N,C
+mistral,N,T,C,T,N,C,N,C
 monasca,N,C,N,C,N,C,N,N
-mongodb,C,C,N,N,C,C,N,N
-multipathd,C,C,C,C,C,C,N,C
-murano,C,C,C,C,C,C,N,C
-neutron,T,T,T,T,T,T,N,T
-nova,T,T,T,T,T,T,N,T
-nova-spicehtml5proxy,T,T,N,N,T,T,N,T
-novajoin,C,C,C,C,N,C,N,C
-octavia,C,C,C,C,N,C,N,C
-opendaylight,C,C,N,N,C,C,N,N
-openvswitch,T,T,T,T,T,T,N,T
-ovn,C,C,C,C,C,C,N,C
+mongodb,N,C,N,N,C,C,N,N
+multipathd,N,C,C,C,C,C,N,C
+murano,N,C,C,C,C,C,N,C
+neutron,N,T,T,T,T,T,N,T
+nova,N,T,T,T,T,T,N,T
+nova-spicehtml5proxy,N,T,N,N,T,T,N,T
+novajoin,N,C,C,C,N,C,N,C
+octavia,N,C,C,C,N,C,N,C
+opendaylight,N,C,N,N,C,C,N,N
+openvswitch,N,T,T,T,T,T,N,T
+ovn,N,C,C,C,C,C,N,C
 ovsdpdk,N,N,N,N,C,C,N,N
-panko,C,C,C,C,C,C,N,C
-placement,T,T,T,T,T,T,N,T
-prometheus,C,C,C,C,C,C,N,C
-ptp,C,C,C,C,C,C,N,C
-qdrouterd,C,C,C,C,N,N,N,N
+panko,N,C,C,C,C,C,N,C
+placement,N,T,T,T,T,T,N,T
+prometheus,N,C,C,C,C,C,N,C
+ptp,N,C,C,C,C,C,N,C
+qdrouterd,N,C,C,C,N,N,N,N
 qinling,N,C,N,C,N,C,N,C
-rabbitmq,T,T,T,T,T,T,N,T
-radvd,C,C,C,C,C,C,N,C
-rally,C,C,C,C,N,C,N,C
-redis,C,T,C,T,C,C,N,C
-rsyslog,C,C,C,C,C,C,N,C
-sahara,C,C,C,C,C,C,N,C
+rabbitmq,N,T,T,T,T,T,N,T
+radvd,N,C,C,C,C,C,N,C
+rally,N,C,C,C,N,C,N,C
+redis,N,T,C,T,C,C,N,C
+rsyslog,N,C,C,C,C,C,N,C
+sahara,N,C,C,C,C,C,N,C
 searchlight,N,C,N,C,N,C,N,C
-senlin,C,C,C,C,C,C,N,C
-sensu,C,C,N,N,N,N,N,N
-skydive,C,C,C,C,C,C,N,C
+senlin,N,C,C,C,C,C,N,C
+sensu,N,C,N,N,N,N,N,N
+skydive,N,C,C,C,C,C,N,C
 solum,N,C,N,C,N,C,N,C
-storm,C,C,C,C,C,C,N,C
-swift,C,C,C,C,C,C,N,C
-tacker,C,T,C,T,N,C,N,C
-telegraf,C,C,C,C,C,C,N,N
-tempest,C,C,C,C,C,C,N,C
-tgtd,C,T,N,N,C,T,N,C
-tripleoclient,C,N,C,N,N,N,N,N
-trove,C,C,C,C,C,C,N,C
-vitrage,C,C,C,C,N,C,N,C
+storm,N,C,C,C,C,C,N,C
+swift,N,C,C,C,C,C,N,C
+tacker,N,T,C,T,N,C,N,C
+telegraf,N,C,C,C,C,C,N,N
+tempest,N,C,C,C,C,C,N,C
+tgtd,N,T,N,N,C,T,N,C
+tripleoclient,N,N,C,N,N,N,N,N
+trove,N,C,C,C,C,C,N,C
+vitrage,N,C,C,C,N,C,N,C
 vmtp,N,C,N,C,N,C,N,C
-watcher,C,C,C,C,C,C,N,C
-zaqar,C,C,C,C,N,C,N,C
-zookeeper,C,C,C,C,C,C,N,C
+watcher,N,C,C,C,C,C,N,C
+zaqar,N,C,C,C,N,C,N,C
+zookeeper,N,C,C,C,C,C,N,C
 zun,N,T,N,T,N,T,N,C

doc/source/support_matrix.rst

@@ -21,17 +21,22 @@ The following base container images are supported:
 
 .. note::
 
-   CentOS 8 is no longer supported since it has been marked as End of Life
+   CentOS Linux 8 is no longer supported since it has been marked as End of Life
    and repositories have been removed from CentOS mirrors.
 
-============= ============ ================
-Distribution  Default base Default base tag
-============= ============ ================
-CentOS 7      centos       7
-Debian Buster debian       10
-RHEL 8        rhel         8
-Ubuntu Bionic ubuntu       18.04
-============= ============ ================
+.. note::
+
+   CentOS Stream 8 is supported only for source type builds.
+
+=============== ============ ================
+Distribution    Default base Default base tag
+=============== ============ ================
+CentOS 7        centos       7
+CentOS Stream 8 centos       stream8
+Debian Buster   debian       10
+RHEL 8          rhel         8
+Ubuntu Bionic   ubuntu       18.04
+=============== ============ ================
 
 The remainder of this document outlines which images are supported on which of
 these distributions.

docker/base/Dockerfile.j2

@@ -93,6 +93,7 @@ COPY yum.conf /etc/yum.conf
     # see https://launchpad.net/bugs/1884034
     {% if distro_package_manager == 'dnf' %}
         {% set base_yum_repo_files = base_yum_repo_files + [
+            'delorean-deps.repo',
             'rabbitmq_rabbitmq-erlang.repo',
         ] %}
     {% endif %}
@@ -225,24 +226,20 @@ RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 {% endblock %}
 
 {% set base_centos_yum_repo_keys = [
-    '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud',
     '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-OpsTools',
     '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Storage',
 ] %}
 
 {% set base_centos_yum_repo_packages = [
-    'centos-release-nfs-ganesha28',
-    'centos-release-openstack-train',
+    'centos-release-ceph-nautilus',
     'centos-release-opstools',
     'epel-release',
 ] %}
-
 {% set base_centos_yum_repos_to_disable = [
 ] %}
 
 {% if distro_package_manager == 'dnf' %}
     {% set base_centos_yum_repo_keys = base_centos_yum_repo_keys + [
-        '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization-RDO',
     ] %}
     # We need 'dnf-plugins-core' for 'dnf config-manager'
     {% set base_centos_yum_repo_packages = base_centos_yum_repo_packages + [
@@ -261,10 +258,12 @@ RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
     ] %}
 {% else %}
     {% set base_centos_yum_repo_keys = base_centos_yum_repo_keys + [
+        '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud',
         '/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization',
     ] %}
     {% set base_centos_yum_repo_packages = base_centos_yum_repo_packages + [
-        'centos-release-ceph-nautilus',
+        'centos-release-nfs-ganesha28',
+        'centos-release-openstack-train',
         'centos-release-qemu-ev',
         'yum-plugin-priorities',
     ] %}
@@ -331,6 +330,7 @@ RUN yum-config-manager --enable rhel-7-server-optional-rpms \
 {% set base_centos_binary_packages = [
         'ca-certificates',
         'findutils',
+        'hostname',
         'iproute',
         'iscsi-initiator-utils',
         'lvm2',
@@ -371,6 +371,7 @@ RUN yum-config-manager --enable rhel-7-server-optional-rpms \
 {% set base_centos_source_packages = [
     'ca-certificates',
     'curl',
+    'hostname',
     'iproute',
     'iscsi-initiator-utils',
     'lvm2',

docker/base/delorean-deps.repo

@@ -0,0 +1,23 @@
+# Adapted from https://trunk.rdoproject.org/centos8-train/delorean-deps.repo
+
+[delorean-train-testing]
+name=dlrn-train-testing
+baseurl=https://trunk.rdoproject.org/centos8-train/deps/latest/
+enabled=1
+gpgcheck=0
+module_hotfixes=1
+
+[delorean-train-build-deps]
+name=dlrn-train-build-deps
+baseurl=https://trunk.rdoproject.org/centos8-train/build-deps/latest/
+enabled=1
+gpgcheck=0
+module_hotfixes=1
+
+[centos-rabbitmq-38]
+name=Messaging RabbitMQ
+baseurl=http://mirror.centos.org/centos/8-stream/messaging/$basearch/rabbitmq-38
+gpgcheck=0
+enabled=1
+module_hotfixes=1
+skip_if_unavailable=1

docker/cinder/cinder-volume/Dockerfile.j2

@@ -15,7 +15,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% if distro_python_version.startswith('3') %}
     {% set cinder_volume_packages = cinder_volume_packages + [
             'python3-rtslib',
-            'python3-cinderlib',
     ] %}
     {% else %}
     {# NOTE(hrw): qpid-proton-c 0.26 from centos-openstack-train repo instead of epel #}

kolla/image/build.py

@@ -176,6 +176,7 @@ UNBUILDABLE_IMAGES = {
     'centos8+source': {
         "bifrost-base",          # Cannot find a valid baseurl for repo: epel
         "cyborg-agent",          # opae-sdk does not support CentOS 8
+        "masakari-monitors",     # Fails to install
     },
 
     'debian': {
@@ -785,7 +786,9 @@ class KollaWorker(object):
 
         if self.base in rh_base and self.base_tag.startswith('7'):
             self.conf.distro_python_version = "2.7"
-        elif self.base in rh_base and self.base_tag.startswith('8'):
+        elif (self.base in rh_base and
+              (self.base_tag.startswith('8') or
+               self.base_tag.startswith('stream8'))):
             self.conf.distro_python_version = "3.6"
         elif self.base in ['debian']:
             self.conf.distro_python_version = "3.7"
@@ -798,7 +801,8 @@ class KollaWorker(object):
         if self.conf.distro_package_manager is not None:
             package_manager = self.conf.distro_package_manager
         elif self.base in rh_base:
-            if LooseVersion(self.base_tag) >= LooseVersion('8'):
+            if (self.base_tag == 'stream8' or
+                    LooseVersion(self.base_tag) >= LooseVersion('8')):
                 package_manager = 'dnf'
             else:
                 package_manager = 'yum'

releasenotes/notes/centos-stream-8-images-afbe2e9f10f7dce8.yaml

@@ -0,0 +1,5 @@
+---
+other:
+  - |
+    CentOS images (only source, not binary) are now buildable using CentOS
+    Stream 8 as base.

tests/templates/kolla-build.conf.j2

@@ -1,5 +1,8 @@
 [DEFAULT]
 debug = true
+{% if ansible_distribution | lower == 'centos' and base_tag | default == 'stream8' %}
+base_image = quay.io/centos/centos
+{% endif %}
 {% if publisher %}
 namespace = kolla
 {# NOTE(hrw): we do not publish centos images for aarch64 so can just override #}
@@ -7,7 +10,7 @@ namespace = kolla
 {% set tag_suffix = '-aarch64' %}
 {% else %}
 {# TODO(mgoddard): Remove tag_suffix when CentOS 7 is no longer supported. #}
-{% set tag_suffix = '-centos8' if ansible_distribution | lower == 'centos' and base_tag | default == 8 else '' %}
+{% set tag_suffix = '-centos8s' if ansible_distribution | lower == 'centos' and base_tag | default == 'stream8' else '' %}
 {% endif %}
 tag = {{ (zuul.tag if zuul.pipeline == "release" else zuul.branch | basename) ~ tag_suffix }}
 {% endif %}

tests/test_build.py

@@ -79,25 +79,27 @@ class BuildTestCentosSource(BuildTest, base.BaseTestCase):
                                 "--type", "source"])
 
 
-class BuildTestCentos8Binary(BuildTest, base.BaseTestCase):
+class BuildTestCentosStream8Binary(BuildTest, base.BaseTestCase):
 
     def setUp(self):
-        super(BuildTestCentos8Binary, self).setUp()
+        super(BuildTestCentosStream8Binary, self).setUp()
         # TODO(mgoddard): Remove --base-tag argument when CentOS 8 is the
         # default.
         self.build_args.extend(["--base", "centos",
-                                "--base-tag", "8",
+                                "--base-tag", "stream8",
+                                "--base-image", "quay.io/centos/centos",
                                 "--type", "binary"])
 
 
-class BuildTestCentos8Source(BuildTest, base.BaseTestCase):
+class BuildTestCentosStream8Source(BuildTest, base.BaseTestCase):
 
     def setUp(self):
-        super(BuildTestCentos8Source, self).setUp()
+        super(BuildTestCentosStream8Source, self).setUp()
         # TODO(mgoddard): Remove --base-tag argument when CentOS 8 is the
         # default.
         self.build_args.extend(["--base", "centos",
-                                "--base-tag", "8",
+                                "--base-tag", "stream8",
+                                "--base-image", "quay.io/centos/centos",
                                 "--type", "source"])
 
 

tox.ini

@@ -103,7 +103,7 @@ commands =
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestCentosSource
 
-[testenv:build-centos8-binary]
+[testenv:build-centosstream8-binary]
 whitelist_externals = find
                       bash
 setenv =
@@ -111,9 +111,9 @@ setenv =
 commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
-  stestr run test_build.BuildTestCentos8Binary
+  stestr run test_build.BuildTestCentosStream8Binary
 
-[testenv:build-centos8-source]
+[testenv:build-centosstream8-source]
 whitelist_externals = find
                       bash
 setenv =
@@ -121,7 +121,7 @@ setenv =
 commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
-  stestr run test_build.BuildTestCentos8Source
+  stestr run test_build.BuildTestCentosStream8Source
 
 [testenv:build-ubuntu-binary]
 whitelist_externals = find