Add CentOS Stream 9 / Rocky Linux 9 host support

Added c9s jobs are non voting, as agreed on PTG to focus on Rocky Linux 9.
Since both CS9 and RL9 have higher default fd limit (1073741816 vs
1048576 in CS8) - lowering that for:
* RMQ - because Erlang allocates memory based on this (see [1], [2], [3]).
* MariaDB - because Galera cluster bootstrap failed

Changed openvswitch_db healthcheck, because for unknown reason
the usual check (using lsof on /run/openvswitch/db.sock) is hanging
on "Bad file descriptor" (even with privileged: true).

[1]: https://github.com/docker-library/rabbitmq/issues/545
[2]: https://github.com/rabbitmq/cluster-operator/issues/959#issuecomment-1043280324
[3]: a8b627aaed

Depends-On: https://review.opendev.org/c/openstack/tenks/+/856296
Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/856328
Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/856443
Needed-By: https://review.opendev.org/c/openstack/kolla/+/836664
Co-Authored-By: Michał Nasiadka <mnasiadka@gmail.com>
Change-Id: I3f7b480519aea38c3927bee7fb2c23eea178554d

ansible/group_vars/all.yml

@@ -152,7 +152,14 @@ docker_common_options:
 # Dimensions options
 ####################
 # Dimension options for Docker Containers
-default_container_dimensions: {}
+# NOTE(mnasiadka): Lower 1073741816 nofile limit on EL9 (RHEL9/CentOS Stream 9/Rocky Linux 9)
+#                  fixes at least rabbitmq and mariadb
+default_container_dimensions: "{{ default_container_dimensions_el9 if ansible_facts.os_family == 'RedHat' else '{}' }}"
+default_container_dimensions_el9:
+  ulimits:
+    nofile:
+      soft: 1048576
+      hard: 1048576
 
 #####################
 # Healthcheck options

ansible/roles/openvswitch/defaults/main.yml

@@ -54,7 +54,7 @@ openvswitch_db_enable_healthchecks: "{{ enable_container_healthchecks }}"
 openvswitch_db_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
 openvswitch_db_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
 openvswitch_db_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
-openvswitch_db_healthcheck_test: ["CMD-SHELL", "healthcheck_socket ovsdb-server /run/openvswitch/db.sock"]
+openvswitch_db_healthcheck_test: ["CMD-SHELL", "ovsdb-client list-dbs"]
 openvswitch_db_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"
 openvswitch_db_healthcheck:
   interval: "{{ openvswitch_db_healthcheck_interval }}"

ansible/roles/prechecks/vars/main.yml

@@ -10,15 +10,15 @@ ansible_version_max: '2.12'
 # for that distribution.
 host_os_distributions:
   CentOS:
-    - "8"
+    - "9"
   Debian:
     - "bullseye"
   openEuler:
     - "22.03"
   RHEL:
-    - "8"
+    - "9"
   Rocky:
-    - "8"
+    - "9"
   Ubuntu:
     - "focal"
     - "jammy"

roles/cephadm/tasks/pkg_redhat.yml

@@ -6,6 +6,12 @@
     recurse: yes
   become: True
 
+- name: Install centos-release-ceph
+  dnf:
+    name: "centos-release-ceph-{{ cephadm_ceph_release }}"
+  become: True
+  when: cephadm_use_package_from_distribution
+
 - name: Enable Ceph base yum repository
   yum_repository:
     name: "{{ item.shortname }}"
@@ -14,3 +20,4 @@
     gpgkey: "{{ cephadm_ceph_rpm_gpg_key }}"
   loop: "{{ cephadm_ceph_rpm_repos }}"
   become: True
+  when: not cephadm_use_package_from_distribution

tests/pre.yml

@@ -41,14 +41,27 @@
   tasks:
     # NOTE(yoctozepto): we use gawk to add time to each logged line
     # outside of Ansible (e.g. for init-runonce)
-    - name: Install gawk and Python modules
+    - name: Install gawk and Python modules (Debian/Ubuntu)
-      package:
+      apt:
         name:
           - gawk
           - python3-pip
           - python3-setuptools
           - python3-wheel
       become: true
+      when: ansible_facts.os_family == 'Debian'
+
+    # NOTE(mnasiadka): python3-wheel is in crb repo for EL9
+    - name: Install gawk and Python modules (RedHat)
+      dnf:
+        enablerepo: "crb"
+        name:
+          - gawk
+          - python3-pip
+          - python3-setuptools
+          - python3-wheel
+      become: true
+      when: ansible_facts.os_family == 'RedHat'
 
     - name: Ensure /tmp/logs/ dir
       file:

tests/run-hashi-vault.yml

@@ -21,6 +21,11 @@
         mode: 0777
       become: true
 
+    - name: ensure python3-pip exists
+      package:
+        name: python3-pip
+      become: true
+
     # NOTE(mgoddard): We need a recent pip to install the latest cryptography
     # library. See https://github.com/pyca/cryptography/issues/5753
     - name: install pip 19.1.1+

zuul.d/jobs.yaml

@@ -1,8 +1,9 @@
 ---
 - job:
-    name: kolla-ansible-centos8s-source
+    name: kolla-ansible-centos9s-source
     parent: kolla-ansible-base
-    nodeset: kolla-ansible-centos8s
+    nodeset: kolla-ansible-centos9s
+    voting: false
     vars:
       base_distro: centos
       tls_enabled: true
@@ -34,10 +35,9 @@
       tls_enabled: true
 
 - job:
-    name: kolla-ansible-rocky8-source
+    name: kolla-ansible-rocky9-source
     parent: kolla-ansible-base
-    nodeset: kolla-ansible-rocky-8
+    nodeset: kolla-ansible-rocky9
-    voting: false
     vars:
       base_distro: centos
       tls_enabled: true
@@ -51,9 +51,9 @@
       tls_enabled: true
 
 - job:
-    name: kolla-ansible-centos8s-source-kvm
+    name: kolla-ansible-rocky9-source-kvm
     parent: kolla-ansible-kvm-base
-    nodeset: kolla-ansible-centos8s-nested-virt
+    nodeset: kolla-ansible-rocky9-nested-virt
     vars:
       base_distro: centos
 
@@ -65,9 +65,9 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-multinode-ipv6
+    name: kolla-ansible-rocky9-source-multinode-ipv6
     parent: kolla-ansible-ipv6-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     vars:
       base_distro: centos
 
@@ -79,12 +79,13 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-cephadm
+    name: kolla-ansible-rocky9-source-cephadm
     parent: kolla-ansible-cephadm-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     timeout: 9000
     vars:
       base_distro: centos
+      cephadm_use_package_from_distribution: true
 
 - job:
     name: kolla-ansible-ubuntu-source-cephadm
@@ -96,9 +97,9 @@
       cephadm_use_package_from_distribution: true
 
 - job:
-    name: kolla-ansible-centos8s-source-mariadb
+    name: kolla-ansible-rocky9-source-mariadb
     parent: kolla-ansible-mariadb-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     vars:
       base_distro: centos
 
@@ -110,9 +111,9 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-upgrade
+    name: kolla-ansible-rocky9-source-upgrade
     parent: kolla-ansible-base
-    nodeset: kolla-ansible-centos8s
+    nodeset: kolla-ansible-rocky9
     timeout: 9000
     vars:
       base_distro: centos
@@ -159,9 +160,9 @@
       tls_enabled: true
 
 - job:
-    name: kolla-ansible-centos8s-source-upgrade-cephadm
+    name: kolla-ansible-rocky9-source-upgrade-cephadm
     parent: kolla-ansible-cephadm-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     timeout: 9000
     vars:
       base_distro: centos
@@ -179,9 +180,9 @@
       is_upgrade: yes
 
 - job:
-    name: kolla-ansible-centos8s-source-upgrade-ovn
+    name: kolla-ansible-rocky9-source-upgrade-ovn
     parent: kolla-ansible-ovn-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     timeout: 9000
     vars:
       base_distro: centos
@@ -199,9 +200,9 @@
       is_upgrade: yes
 
 - job:
-    name: kolla-ansible-centos8s-source-bifrost
+    name: kolla-ansible-rocky9-source-bifrost
     parent: kolla-ansible-bifrost-base
-    nodeset: kolla-ansible-centos8s
+    nodeset: kolla-ansible-rocky9
     vars:
       base_distro: centos
 
@@ -213,9 +214,9 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-zun
+    name: kolla-ansible-rocky9-source-zun
     parent: kolla-ansible-zun-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     vars:
       base_distro: centos
 
@@ -234,9 +235,9 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-swift
+    name: kolla-ansible-rocky9-source-swift
     parent: kolla-ansible-swift-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     vars:
       base_distro: centos
 
@@ -248,9 +249,9 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-swift-upgrade
+    name: kolla-ansible-rocky9-source-swift-upgrade
     parent: kolla-ansible-swift-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     timeout: 10800
     vars:
       base_distro: centos
@@ -268,16 +269,16 @@
       is_upgrade: yes
 
 - job:
-    name: kolla-ansible-centos8s-source-scenario-nfv
+    name: kolla-ansible-rocky9-source-scenario-nfv
     parent: kolla-ansible-scenario-nfv-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     vars:
       base_distro: centos
 
 - job:
-    name: kolla-ansible-centos8s-source-ironic
+    name: kolla-ansible-rocky9-source-ironic
     parent: kolla-ansible-ironic-base
-    nodeset: kolla-ansible-centos8s
+    nodeset: kolla-ansible-rocky9
     vars:
       base_distro: centos
 
@@ -296,9 +297,9 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-magnum
+    name: kolla-ansible-rocky9-source-magnum
     parent: kolla-ansible-magnum-base
-    nodeset: kolla-ansible-centos8s
+    nodeset: kolla-ansible-rocky9
     vars:
       base_distro: centos
 
@@ -310,9 +311,9 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-octavia
+    name: kolla-ansible-rocky9-source-octavia
     parent: kolla-ansible-octavia-base
-    nodeset: kolla-ansible-centos8s
+    nodeset: kolla-ansible-rocky9
     vars:
       base_distro: centos
 
@@ -331,16 +332,16 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-masakari
+    name: kolla-ansible-rocky9-source-masakari
     parent: kolla-ansible-masakari-base
-    nodeset: kolla-ansible-centos8s-masakari
+    nodeset: kolla-ansible-rocky9-masakari
     vars:
       base_distro: centos
 
 - job:
-    name: kolla-ansible-centos8s-source-monasca
+    name: kolla-ansible-rocky9-source-monasca
     parent: kolla-ansible-monasca-base
-    nodeset: kolla-ansible-centos8s-multi-monasca
+    nodeset: kolla-ansible-rocky9-multi-monasca
     vars:
       base_distro: centos
 
@@ -352,16 +353,16 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-cells
+    name: kolla-ansible-rocky9-source-cells
     parent: kolla-ansible-cells-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     vars:
       base_distro: centos
 
 - job:
-    name: kolla-ansible-centos8s-source-ovn
+    name: kolla-ansible-rocky9-source-ovn
     parent: kolla-ansible-ovn-base
-    nodeset: kolla-ansible-centos8s-multi
+    nodeset: kolla-ansible-rocky9-multi
     vars:
       base_distro: centos
 
@@ -373,9 +374,9 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-prometheus-efk
+    name: kolla-ansible-rocky9-source-prometheus-efk
     parent: kolla-ansible-prometheus-efk-base
-    nodeset: kolla-ansible-centos8s
+    nodeset: kolla-ansible-rocky9
     vars:
       base_distro: centos
 
@@ -387,9 +388,9 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-source-venus
+    name: kolla-ansible-rocky9-source-venus
     parent: kolla-ansible-venus-base
-    nodeset: kolla-ansible-centos8s
+    nodeset: kolla-ansible-rocky9
     vars:
       base_distro: centos
 
@@ -401,8 +402,8 @@
       base_distro: ubuntu
 
 - job:
-    name: kolla-ansible-centos8s-hashi-vault
+    name: kolla-ansible-rocky9-hashi-vault
     parent: kolla-ansible-hashi-vault-base
-    nodeset: kolla-ansible-centos8s
+    nodeset: kolla-ansible-rocky9
     vars:
       base_distro: centos

zuul.d/nodesets.yaml

@@ -1,9 +1,9 @@
 ---
 - nodeset:
-    name: kolla-ansible-centos8s
+    name: kolla-ansible-centos9s
     nodes:
       - name: primary
-        label: centos-8-stream
+        label: centos-9-stream
 
 - nodeset:
     name: kolla-ansible-debian-bullseye
@@ -46,10 +46,10 @@
         label: openEuler-22-03-LTS
 
 - nodeset:
-    name: kolla-ansible-rocky-8
+    name: kolla-ansible-rocky9
     nodes:
       - name: primary
-        label: rockylinux-8
+        label: rockylinux-9
 
 - nodeset:
     name: kolla-ansible-focal-multi
@@ -72,14 +72,14 @@
         label: ubuntu-jammy
 
 - nodeset:
-    name: kolla-ansible-centos8s-multi
+    name: kolla-ansible-rocky9-multi
     nodes:
       - name: primary
-        label: centos-8-stream
+        label: rockylinux-9
       - name: secondary1
-        label: centos-8-stream
+        label: rockylinux-9
       - name: secondary2
-        label: centos-8-stream
+        label: rockylinux-9
 
 - nodeset:
     name: kolla-ansible-jammy-nested-virt
@@ -88,26 +88,26 @@
         label: nested-virt-ubuntu-jammy
 
 - nodeset:
-    name: kolla-ansible-centos8s-nested-virt
+    name: kolla-ansible-rocky9-nested-virt
     nodes:
       - name: primary
-        label: nested-virt-centos-8-stream
+        label: nested-virt-centos-9-stream
 
 - nodeset:
-    name: kolla-ansible-centos8s-multi-monasca
+    name: kolla-ansible-rocky9-multi-monasca
     nodes:
       - name: primary
-        label: centos-8-stream
+        label: rockylinux-9
       - name: secondary1
-        label: centos-8-stream
+        label: rockylinux-9
       - name: secondary2
-        label: centos-8-stream
+        label: rockylinux-9
       - name: secondary3
-        label: centos-8-stream
+        label: rockylinux-9
       - name: secondary4
-        label: centos-8-stream
+        label: rockylinux-9
       - name: secondary5
-        label: centos-8-stream
+        label: rockylinux-9
 
 - nodeset:
     name: kolla-ansible-jammy-masakari
@@ -122,13 +122,13 @@
         label: ubuntu-jammy
 
 - nodeset:
-    name: kolla-ansible-centos8s-masakari
+    name: kolla-ansible-rocky9-masakari
     nodes:
       - name: primary
-        label: centos-8-stream
+        label: rockylinux-9
       - name: secondary
-        label: centos-8-stream
+        label: rockylinux-9
       - name: ternary1
-        label: centos-8-stream
+        label: rockylinux-9
       - name: ternary2
-        label: centos-8-stream
+        label: rockylinux-9

zuul.d/project.yaml

@@ -11,67 +11,66 @@
       - periodic-stable-jobs
     check:
       jobs:
-        # - kolla-ansible-centos8s-source
+        - kolla-ansible-centos9s-source
         - kolla-ansible-debian-source
         - kolla-ansible-openeuler-source
-        # - kolla-ansible-rocky8-source
+        - kolla-ansible-rocky9-source
         - kolla-ansible-ubuntu-source
-        # - kolla-ansible-centos8s-source-kvm
+        - kolla-ansible-rocky9-source-kvm
         - kolla-ansible-ubuntu-source-kvm
-        # - kolla-ansible-centos8s-source-multinode-ipv6
+        - kolla-ansible-rocky9-source-multinode-ipv6
         - kolla-ansible-ubuntu-source-multinode-ipv6
-        # - kolla-ansible-centos8s-source-bifrost
+        - kolla-ansible-rocky9-source-bifrost
         - kolla-ansible-ubuntu-source-bifrost
-        # - kolla-ansible-centos8s-source-zun
+        - kolla-ansible-rocky9-source-zun
         - kolla-ansible-debian-source-zun
         - kolla-ansible-ubuntu-source-zun
-        # - kolla-ansible-centos8s-source-swift
+        - kolla-ansible-rocky9-source-swift
         - kolla-ansible-ubuntu-source-swift
-        # - kolla-ansible-centos8s-source-scenario-nfv
+        - kolla-ansible-rocky9-source-scenario-nfv
-        # - kolla-ansible-centos8s-source-magnum
+        - kolla-ansible-rocky9-source-magnum
         - kolla-ansible-ubuntu-source-magnum
-        # - kolla-ansible-centos8s-source-octavia
+        - kolla-ansible-rocky9-source-octavia
         - kolla-ansible-ubuntu-source-octavia
-        # - kolla-ansible-centos8s-source-masakari
+        - kolla-ansible-rocky9-source-masakari
         - kolla-ansible-ubuntu-source-masakari
-        # - kolla-ansible-centos8s-source-ironic
+        - kolla-ansible-rocky9-source-ironic
         - kolla-ansible-debian-source-ironic
         - kolla-ansible-ubuntu-source-ironic
-        # - kolla-ansible-centos8s-source-upgrade
+        # - kolla-ansible-rocky9-source-upgrade
         - kolla-ansible-debian-source-upgrade
         - kolla-ansible-ubuntu-source-upgrade
         - kolla-ansible-ubuntu-binary-upgrade
-        # - kolla-ansible-centos8s-source-cells
         - kolla-ansible-ubuntu-source-cells
-        # - kolla-ansible-centos8s-source-mariadb
+        - kolla-ansible-rocky9-source-cells
+        - kolla-ansible-rocky9-source-mariadb
         - kolla-ansible-ubuntu-source-mariadb
-        # - kolla-ansible-centos8s-source-ovn
+        - kolla-ansible-rocky9-source-ovn
         - kolla-ansible-ubuntu-source-ovn
-        # - kolla-ansible-centos8s-source-upgrade-ovn
+        # - kolla-ansible-rocky9-source-upgrade-ovn
         - kolla-ansible-ubuntu-source-upgrade-ovn
-        # - kolla-ansible-centos8s-source-prometheus-efk
+        - kolla-ansible-rocky9-source-prometheus-efk
         - kolla-ansible-ubuntu-source-prometheus-efk
-        # - kolla-ansible-centos8s-source-venus
+        - kolla-ansible-rocky9-source-venus
         - kolla-ansible-ubuntu-source-venus
-        # - kolla-ansible-centos8s-source-monasca
+        - kolla-ansible-rocky9-source-cephadm
-        # - kolla-ansible-centos8s-source-cephadm
         - kolla-ansible-ubuntu-source-cephadm
-        # - kolla-ansible-centos8s-source-upgrade-cephadm
+        # - kolla-ansible-rocky9-source-upgrade-cephadm
         - kolla-ansible-ubuntu-source-upgrade-cephadm
-        # - kolla-ansible-centos8s-hashi-vault
+        - kolla-ansible-rocky9-hashi-vault
     check-arm64:
       jobs:
         - kolla-ansible-debian-source-aarch64
         - kolla-ansible-debian-source-upgrade-aarch64
     gate:
       jobs:
-        # - kolla-ansible-centos8s-source
+        # - kolla-ansible-rocky9-source-upgrade
-        # - kolla-ansible-centos8s-source-upgrade
         - kolla-ansible-debian-source
         - kolla-ansible-debian-source-upgrade
+        - kolla-ansible-rocky9-source
         - kolla-ansible-ubuntu-source
         - kolla-ansible-ubuntu-source-upgrade
     experimental:
       jobs:
-        # - kolla-ansible-centos8s-source-swift-upgrade
+        - kolla-ansible-rocky9-source-swift-upgrade
         - kolla-ansible-ubuntu-source-swift-upgrade