From 1f5014e01104c2ccf275c2ed082b6861c331ac65 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Wed, 21 Apr 2021 19:41:45 +0200 Subject: [PATCH] move to Debian 'bullseye' Bullseye is now in hard freeze cycle and goes for release. https://release.debian.org/bullseye/freeze_policy.html Co-Authored-By: Michal Nasiadka Change-Id: I543965a2741cebfa759576a4c75669a7bacd4208 (cherry-pick from master e0f8cbc50cf7184930a501b0f5452a64f523cd6f) --- .zuul.d/debian.yaml | 8 ++- doc/source/matrix_aarch64.csv | 8 +-- doc/source/matrix_x86.csv | 6 +- docker/base/Dockerfile.j2 | 14 +++- docker/base/apt_preferences.debian | 47 +------------ docker/base/sources.list.debian | 31 +-------- docker/mariadb/mariadb-server/Dockerfile.j2 | 4 ++ .../security_reset.expect.debian | 67 +++++++++++++++++++ .../openvswitch-base/Dockerfile.j2 | 3 +- kolla/common/config.py | 4 +- kolla/image/build.py | 16 ++--- kolla/template/repos.yaml | 2 - kolla/tests/test_build.py | 2 +- tests/templates/template_overrides.j2 | 2 +- 14 files changed, 112 insertions(+), 102 deletions(-) create mode 100644 docker/mariadb/mariadb-server/security_reset.expect.debian diff --git a/.zuul.d/debian.yaml b/.zuul.d/debian.yaml index 0ae1ef5de5..b1a32dac0c 100644 --- a/.zuul.d/debian.yaml +++ b/.zuul.d/debian.yaml @@ -4,11 +4,15 @@ jobs: - kolla-build-debian-source - kolla-build-debian-binary - - kolla-ansible-debian-source + - kolla-ansible-debian-source: + vars: + kolla_python_version: 3.9 check-arm64: jobs: - kolla-build-debian-source-aarch64 - - kolla-ansible-debian-source-aarch64 + - kolla-ansible-debian-source-aarch64: + vars: + kolla_python_version: 3.9 periodic: jobs: - kolla-publish-debian-source-quay diff --git a/doc/source/matrix_aarch64.csv b/doc/source/matrix_aarch64.csv index e7d4bfcefd..9b4ed7a4d2 100644 --- a/doc/source/matrix_aarch64.csv +++ b/doc/source/matrix_aarch64.csv @@ -32,14 +32,14 @@ iscsid,C,C,C,C,N,C kafka,C,C,C,C,N,C keepalived,C,C,C,C,N,C keystone,C,C,C,C,N,C -kibana,N,N,N,N,C,C +kibana,N,N,N,N,N,C kolla-toolbox,C,C,C,C,N,C kuryr,N,C,N,C,N,C logstash,C,C,C,C,N,C magnum,C,C,C,C,N,C manila,C,C,C,C,N,C mariadb,C,C,C,C,N,C -masakari,N,C,C,C,C,C +masakari,N,C,C,C,N,C memcached,C,C,C,C,N,C mistral,C,C,C,C,N,C monasca,N,N,N,N,N,N @@ -51,8 +51,8 @@ nova,C,C,C,C,N,C nova-spicehtml5proxy,N,N,C,C,N,C octavia,C,C,N,C,N,C openvswitch,C,C,C,C,N,C -ovn,C,C,C,C,N,C -ovsdpdk,N,N,C,C,N,N +ovn,C,C,C,C,N,N +ovsdpdk,N,N,C,C,N,C panko,C,C,C,C,N,C placement,C,C,N,C,N,C prometheus,C,C,C,C,N,C diff --git a/doc/source/matrix_x86.csv b/doc/source/matrix_x86.csv index 89b20adfb9..5a423ca439 100644 --- a/doc/source/matrix_x86.csv +++ b/doc/source/matrix_x86.csv @@ -47,13 +47,13 @@ monasca,N,C,N,C,N,N multipathd,C,C,C,C,C,C murano,C,C,C,C,C,C neutron,T,T,T,T,C,T -neutron-mlnx-agent,C,C,N,C,N,C +neutron-mlnx-agent,C,C,N,C,C,C nova,T,T,T,T,C,T nova-spicehtml5proxy,N,N,T,T,C,T octavia,C,C,N,C,C,C openvswitch,T,T,T,T,C,T -ovn,C,C,C,C,C,C -ovsdpdk,N,N,C,C,N,N +ovn,C,C,C,C,N,N +ovsdpdk,N,N,C,C,C,C panko,C,C,C,C,C,C placement,T,T,T,T,C,T prometheus,C,C,C,C,C,C diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index fc55949eeb..6c5d9df832 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -328,6 +328,18 @@ COPY sources.list /etc/apt/sources.list.d/kolla-custom.list {% block base_debian_after_sources_list %}{% endblock %} +{# install Debian Openstack repos - they are not mirrored on CI #} +{% if base_distro == 'debian' %} + +RUN apt update \ + && apt install -y --no-install-recommends extrepo \ + && extrepo enable openstack_wallaby \ + && apt purge -y extrepo \ + && apt --purge autoremove -y \ + && apt clean + +{% endif %} + {% block base_ubuntu_package_apt_preferences %} COPY apt_preferences.{{ base_distro }} /etc/apt/preferences COPY apt_preferences /etc/apt/preferences.d/kolla-custom @@ -383,10 +395,8 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom {% set remote_apt_keys = [ 'https://download.docker.com/linux/debian/gpg', 'https://packages.grafana.com/gpg.key', - 'http://buster-ussuri.debian.net/debian/dists/pubkey.gpg', 'https://packages.treasuredata.com/GPG-KEY-td-agent', 'https://repos.influxdata.com/influxdb.key', - 'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster/Release.key', ] %} {% endif %} diff --git a/docker/base/apt_preferences.debian b/docker/base/apt_preferences.debian index ef693a5d4f..e3f7bb7902 100644 --- a/docker/base/apt_preferences.debian +++ b/docker/base/apt_preferences.debian @@ -1,53 +1,8 @@ -# We want Ceph/nautilus -Package: ceph* libceph* librados* librbd* librgw* python3-ceph* python3-rados python3-rbd python3-rgw radosgw -Pin: version 14.* -Pin-Priority: 1000 - -# ceph-osd requires smartmontools from backports -Package: smartmontools -Pin: version 7.* -Pin-Priority: 1000 - Package: rabbitmq-server Pin: version 3.8.* Pin-Priority: 1000 -# We need erlang 22+ for rabbitmq (aarch64 has v23, x86-64 has v22 and v23 so we use v23) +# RabbitMQ wants 23+ and we want to stay on 23 if possible Package: erlang* Pin: version 1:23.* Pin-Priority: 1000 - -# We do not want packages from this repo -Package: * -Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster -Pin-Priority: 100 - -# We need libvirt 7.0.0 for nova -Package: *libvirt* -Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster -Pin-Priority: 600 - -# on ppc64le we lack libvirt 7.0.0 so let fail with Debian one -Package: *libvirt* -Pin: release o=Debian,b=ppc64le -Pin-Priority: 700 - -# ovn-host depends on openvswitch from buster -Package: openvswitch* -Pin: release n=buster -Pin-Priority: 700 - -# we want newer QEMU -Package: qemu* -Pin: version 1:5.* -Pin-Priority: 700 - -# QEMU depends on those -Package: libpmem1 libndctl6 libdaxctl1 pmdk-tools libpmemblk1 libpmemlog1 libpmemobj1 libpmempool1 liburing1 libfdt1 -Pin: release n=buster-backports -Pin-Priority: 700 - -# nova wants data in /usr/share/qemu/firmware -Package: qemu-efi-aarch64 -Pin: release n=buster-backports -Pin-Priority: 700 diff --git a/docker/base/sources.list.debian b/docker/base/sources.list.debian index d74f36c332..6b975b67c0 100644 --- a/docker/base/sources.list.debian +++ b/docker/base/sources.list.debian @@ -1,33 +1,8 @@ # Default repos -deb http://deb.debian.org/debian buster main +deb http://deb.debian.org/debian bullseye main # debian security updates -deb http://security.debian.org/debian-security buster/updates main +deb http://deb.debian.org/debian-security bullseye-security main # debian backports -deb http://deb.debian.org/debian buster-backports main - -# Buster - Victoria repos -deb http://buster-victoria.debian.net/debian buster-victoria-backports main -deb http://buster-victoria.debian.net/debian buster-victoria-backports-nochange main - -# NOTE(hrw): repositories below are added into image when they are needed as -# separate files in /etc/apt/sources.list.d/ directory. For that purpose they -# are defined in kolla/repos.yaml file. -# -# Here they stay in case someone needs them. - -# elasticsearch (arch:all), logstash (arch:all), kibana (arch:amd64) -#deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main - -# Grafana repo -#deb https://packages.grafana.com/oss/deb stable main - -# td-agent for fluentd -#deb http://packages.treasuredata.com/3/debian/buster buster contrib - -# rabbitmq repo -#deb https://dl.bintray.com/rabbitmq/debian/ buster main - -# erlang repo -#deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang +deb http://deb.debian.org/debian bullseye-backports main diff --git a/docker/mariadb/mariadb-server/Dockerfile.j2 b/docker/mariadb/mariadb-server/Dockerfile.j2 index b96e922feb..9ee7ef37da 100644 --- a/docker/mariadb/mariadb-server/Dockerfile.j2 +++ b/docker/mariadb/mariadb-server/Dockerfile.j2 @@ -37,7 +37,11 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build COPY mariadb_sudoers /etc/sudoers.d/kolla_mariadb_sudoers COPY extend_start.sh /usr/local/bin/kolla_extend_start +{% if base_distro == 'debian' %} +COPY security_reset.expect.debian /usr/local/bin/kolla_security_reset +{% else %} COPY security_reset.expect /usr/local/bin/kolla_security_reset +{% endif %} RUN chmod 755 /usr/local/bin/kolla_extend_start \ && chmod 755 /usr/local/bin/kolla_security_reset \ && chmod 750 /etc/sudoers.d \ diff --git a/docker/mariadb/mariadb-server/security_reset.expect.debian b/docker/mariadb/mariadb-server/security_reset.expect.debian new file mode 100644 index 0000000000..ec40eaf903 --- /dev/null +++ b/docker/mariadb/mariadb-server/security_reset.expect.debian @@ -0,0 +1,67 @@ +#!/usr/bin/expect -f + +if [catch {set timeout $env(DB_MAX_TIMEOUT)}] {set timeout 10} +spawn mysql_secure_installation +expect { + timeout { send_user "\nFailed to get 'Enter current password for root (enter for none):' prompt\n"; exit 1 } + eof { send_user "\nFailed to get 'Enter current password for root (enter for none):' prompt\n"; exit 1 } + "Enter current password for root (enter for none):" +} +send "\r" + +expect { + timeout { send_user "\nFailed to get 'Switch to unix_socket authentication [Y/n] ' prompt\n"; exit 1 } + eof { send_user "\nFailed to get 'Switch to unix_socket authentication' prompt\n"; exit 1 } + "Switch to unix_socket authentication \\\[Y/n\\\] " +} +send "n\r" + +expect { + timeout { send_user "\nFailed to get 'Change the root password? [Y/n]' prompt\n"; exit 1 } + eof { send_user "\nFailed to get 'Change the root password?' prompt\n"; exit 1 } + "Change the root password? \\\[Y/n\\\] " +} +send "y\r" + +expect { + timeout { send_user "\nFailed to get 'New password:' prompt\n"; exit 1 } + eof { send_user "\nFailed to get 'New password:' prompt\n"; exit 1 } + "New password:" +} +send "$env(DB_ROOT_PASSWORD)\r" + +expect { + timeout { send_user "\nFailed to get 'Re-enter new password:' prompt\n"; exit 1 } + eof { send_user "\nFailed to get 'Re-enter new password:' prompt\n"; exit 1 } + "Re-enter new password:" +} +send "$env(DB_ROOT_PASSWORD)\r" + +expect { + timeout { send_user "\nFailed to get 'Remove anonymous users?' prompt\n"; exit 1 } + eof { send_user "\nFailed to get 'Remove anonymous users?' prompt\n"; exit 1 } + "Remove anonymous users?" +} +send "y\r" + +expect { + timeout { send_user "\nFailed to get 'Disallow root login remotely?' prompt\n"; exit 1 } + eof { send_user "\nFailed to get 'Disallow root login remotely?' prompt\n"; exit 1 } + "Disallow root login remotely?" +} +send "n\r" + +expect { + timeout { send_user "\nFailed to get 'Remove test database and access to it?' prompt\n"; exit 1 } + eof { send_user "\nFailed to get 'Remove test database and access to it?' prompt\n"; exit 1 } + "Remove test database and access to it?" +} +send "y\r" + +expect { + timeout { send_user "\nFailed to get 'Reload privilege tables now?' prompt\n"; exit 1 } + eof { send_user "\nFailed to get 'Reload privilege tables now?' prompt\n"; exit 1 } + "Reload privilege tables now?" +} +send "y\r" +expect eof diff --git a/docker/openvswitch/openvswitch-base/Dockerfile.j2 b/docker/openvswitch/openvswitch-base/Dockerfile.j2 index 5bc40ec471..1ddf669e9b 100644 --- a/docker/openvswitch/openvswitch-base/Dockerfile.j2 +++ b/docker/openvswitch/openvswitch-base/Dockerfile.j2 @@ -34,7 +34,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build COPY extend_start.sh /usr/local/bin/kolla_extend_start -RUN mv /usr/sbin/tcpdump /usr/bin/tcpdump \ +# NOTE(hrw): Upstream moved tcpdump to /usr/bin/ in 4.99 version (used in Debian) +RUN if [ -e /usr/sbin/tcpdump ]; then mv /usr/sbin/tcpdump /usr/bin/tcpdump; fi \ && touch /usr/local/bin/kolla_openvswitch_extend_start \ && chmod 755 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_openvswitch_extend_start diff --git a/kolla/common/config.py b/kolla/common/config.py index 96701e72d1..23e5ead402 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -24,13 +24,13 @@ BASE_ARCH = ['x86_64', 'ppc64le', 'aarch64'] DEFAULT_BASE_TAGS = { 'centos': {'name': 'quay.io/centos/centos', 'tag': 'stream8'}, 'rhel': {'name': 'registry.access.redhat.com/ubi8', 'tag': 'latest'}, - 'debian': {'name': 'debian', 'tag': '10'}, + 'debian': {'name': 'debian', 'tag': 'bullseye'}, 'ubuntu': {'name': 'ubuntu', 'tag': '20.04'}, } DISTRO_RELEASE = { 'centos': '8', 'rhel': '8', - 'debian': '10', + 'debian': 'testing/unstable', 'ubuntu': '20.04', } OPENSTACK_RELEASE = 'wallaby' diff --git a/kolla/image/build.py b/kolla/image/build.py index a148c980fb..27006e1acd 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -132,7 +132,7 @@ UNBUILDABLE_IMAGES = { "bifrost-base", # tries to install 'mysql-server' which is not in # Debian 'buster' "monasca-grafana", # FIXME(hrw): some ssl issues to fix - "ovsdpdk", + "ovn-base", # needs more checking "qdrouterd", }, @@ -162,14 +162,10 @@ UNBUILDABLE_IMAGES = { }, 'debian+binary': { - "cloudkitty-base", - "ironic-neutron-agent", - "nova-serialproxy", - "senlin-conductor", # no binary package - "senlin-health-manager", # no binary package - "tacker-base", - "trove-base", # package is not installable - "neutron-mlnx-agent", + "cloudkitty-base", # no support in Dockerfile + "ironic-neutron-agent", # no support in Dockerfile + "nova-serialproxy", # no binary package + "tacker-base", # no binary package }, 'ubuntu+binary': { @@ -697,7 +693,7 @@ class KollaWorker(object): if self.base in rh_base: self.conf.distro_python_version = "3.6" elif self.base in ['debian']: - self.conf.distro_python_version = "3.7" + self.conf.distro_python_version = "3.9" elif self.base in ['ubuntu']: self.conf.distro_python_version = "3.8" else: diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index 05fb8bcaf0..f522a9e85c 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -63,7 +63,6 @@ debian: erlang: "deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/debian buster stable" - libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz:/kolla-wallaby/debian-buster ./" logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" kibana: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main" @@ -76,7 +75,6 @@ debian-aarch64: influxdb: "deb https://repos.influxdata.com/debian buster stable" logstash: "deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main" kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-7.x/apt stable main" - libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz:/kolla-wallaby/debian-buster ./" rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/debian/ buster main" td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib" diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index 8dd34f2fd8..10d70ce5a8 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -529,7 +529,7 @@ class KollaWorkerTest(base.TestCase): """check distro_python_version for Debian""" self.conf.set_override('base', 'debian') kolla = build.KollaWorker(self.conf) - self.assertEqual('3.7', kolla.distro_python_version) + self.assertEqual('3.9', kolla.distro_python_version) def test_build_distro_python_version_rhel80(self): """check distro_python_version for RHEL8.0""" diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2 index d56c601e1c..6be8b794be 100644 --- a/tests/templates/template_overrides.j2 +++ b/tests/templates/template_overrides.j2 @@ -18,7 +18,7 @@ RUN echo registry={{ nodepool_npmjs_proxy }} > /etc/npmrc \ {% if base_distro in ['debian', 'ubuntu'] %} {% endraw %} -# NOTE(hrw): Debian 'buster' and Ubuntu 18.04 refuse to use unsigned repos +# NOTE(hrw): Debian since 'buster' and Ubuntu since 18.04 refuse to use unsigned repos RUN echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated \ && echo 'Acquire::AllowInsecureRepositories "true";' > /etc/apt/apt.conf.d/99allow-insecure-repos