Upgrade RabbitMQ to 3.8

Upgrade and pin RabbitMQ to 3.8.

Upgrade Erlang to 22 to meet requirements:
https://www.rabbitmq.com/which-erlang.html

Start using bintray repository provided by RabbitMQ.

Linaro builds will be used for aarch64.

Implements: blueprint rabbitmq-version-upgrade

Closes-Bug: #1848452

Change-Id: I3d775fd832f2603181da89009cebc1370a61a023

docker/base/Dockerfile.j2

@@ -93,6 +93,7 @@ RUN {{ macros.install_packages(base_centos_language_packages | customizable("cen
         'grafana.repo',
         'influxdb.repo',
         'rabbitmq_rabbitmq-server.repo',
+        'rabbitmq_rabbitmq-erlang.repo',
         'td.repo',
     ] %}
     # FIXME(mgoddard): Not available for CentOS 8 yet.
@@ -107,6 +108,7 @@ RUN {{ macros.install_packages(base_centos_language_packages | customizable("cen
     {% set base_yum_repo_keys = [
         'https://packages.grafana.com/gpg.key',
         'https://repos.influxdata.com/influxdb.key',
+        'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc',
         'https://packages.treasuredata.com/GPG-KEY-td-agent',
     ] %}
     # FIXME(mgoddard): Not available for CentOS 8 yet.
@@ -119,6 +121,7 @@ RUN {{ macros.install_packages(base_centos_language_packages | customizable("cen
     {% set base_yum_repo_files = [
         'grafana.repo',
         'rabbitmq_rabbitmq-server.repo',
+        'rabbitmq_rabbitmq-erlang-aarch64.repo',
     ] %}
     # FIXME(mgoddard): Not available for CentOS 8 yet.
     {% if distro_package_manager == 'yum' %}
@@ -129,6 +132,8 @@ RUN {{ macros.install_packages(base_centos_language_packages | customizable("cen
 
     {% set base_yum_repo_keys = [
         'https://packages.grafana.com/gpg.key',
+        'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc',
+        'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/centos7/repodata/repomd.xml.key',
     ] %}
     # FIXME(mgoddard): Not available for CentOS 8 yet.
     {% if distro_package_manager == 'yum' %}
@@ -437,10 +442,15 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
       'F1656F24C74CD1D8',
     ] %}
     {% set remote_apt_keys = [
-      'https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey',
+      'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc',
       'https://packages.grafana.com/gpg.key',
-      'https://repos.influxdata.com/influxdb.key'
+      'https://repos.influxdata.com/influxdb.key',
     ] %}
+    {% if base_arch == 'aarch64' %}
+        {% set remote_apt_keys = remote_apt_keys + [
+          'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-bionic/Release.key',
+        ] %}
+    {% endif %}
 {% elif base_distro == 'debian' %}
     {% set base_apt_keys = [
       '46095ACC8548582C1A2699A9D27D666CD88E42B4',
@@ -452,7 +462,13 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
       'https://packages.treasuredata.com/GPG-KEY-td-agent',
       'https://repos.influxdata.com/influxdb.key',
       'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster/Release.key',
+      'https://www.rabbitmq.com/rabbitmq-release-signing-key.asc',
     ] %}
+    {% if base_arch == 'aarch64' %}
+        {% set remote_apt_keys = remote_apt_keys + [
+          'https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster/Release.key',
+        ] %}
+    {% endif %}
     {% set base_apt_packages = base_apt_packages +
       ['sudo',]
     %}

docker/base/apt_preferences.debian

@@ -8,6 +8,11 @@ Package: smartmontools
 Pin: version 7.*
 Pin-Priority: 1000
 
+# We do not pin minor versions anymore
+Package: rabbitmq-server
+Pin: version 3.8.*
+Pin-Priority: 1000
+
 # We do not want packages from this repo
 Package: *
 Pin: release o=obs://private/home:marcin.juszkiewicz/debian-buster

docker/base/apt_preferences.ubuntu

@@ -1,3 +1,4 @@
+# We do not pin minor versions anymore
 Package: rabbitmq-server
-Pin: version 3.7.10-1
+Pin: version 3.8.*
 Pin-Priority: 1000

docker/base/rabbitmq_rabbitmq-erlang-aarch64.repo

@@ -0,0 +1,7 @@
+[rabbitmq_rabbitmq-erlang]
+name=rabbitmq_rabbitmq-erlang
+baseurl=http://obs.linaro.org/home:/marcin.juszkiewicz/centos$releasever/
+repo_gpgcheck=0
+gpgcheck=1
+enabled=0
+gpgkey=http://obs.linaro.org/home:/marcin.juszkiewicz/centos$releasever/repodata/repomd.xml.key

docker/base/rabbitmq_rabbitmq-erlang.repo

@@ -0,0 +1,7 @@
+[rabbitmq_rabbitmq-erlang]
+name=rabbitmq_rabbitmq-erlang
+baseurl=https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/22/el/$releasever/
+repo_gpgcheck=0
+gpgcheck=1
+enabled=0
+gpgkey=https://www.rabbitmq.com/rabbitmq-release-signing-key.asc

docker/base/rabbitmq_rabbitmq-server.repo

@@ -1,13 +1,11 @@
 [rabbitmq_rabbitmq-server]
 name=rabbitmq_rabbitmq-server
-baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/$releasever/$basearch
+baseurl=https://dl.bintray.com/rabbitmq/rpm/rabbitmq-server/v3.8.x/el/$releasever/
 # NOTE(mgoddard): rabbitmq-server 3.7 packages are not signed. Previously
 # repo_gpgcheck was set to 1, but this breaks DNF which does not seem to accept
 # keys imported by rpm --import for signing a repo (as opposed to packages).
 # TODO(mgoddard): Set gpgcheck=1 for rabbitmq-server 3.8+
-gpgcheck=0
+repo_gpgcheck=0
+gpgcheck=1
 enabled=0
-gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey
+gpgkey=https://www.rabbitmq.com/rabbitmq-release-signing-key.asc
-sslverify=1
-sslcacert=/etc/pki/tls/certs/ca-bundle.crt
-metadata_expire=300

docker/base/sources.list.debian

@@ -25,3 +25,12 @@ deb http://buster-train.debian.net/debian buster-train-backports-nochange main
 
 # td-agent for fluentd
 #deb http://packages.treasuredata.com/3/debian/buster buster contrib
+
+# rabbitmq repo
+#deb https://dl.bintray.com/rabbitmq/debian/ buster main
+
+# erlang repo
+#deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang
+
+# erlang repo - aarch64 only
+#deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./

docker/base/sources.list.ubuntu

@@ -40,7 +40,13 @@ deb http://ubuntu-cloud.archive.canonical.com/ubuntu bionic-updates/ussuri main
 #deb http://ppa.launchpad.net/odl-team/carbon/ubuntu xenial main
 
 # rabbitmq repo
-#deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main
+#deb https://dl.bintray.com/rabbitmq/debian/ bionic main
+
+# erlang repo
+#deb https://dl.bintray.com/rabbitmq-erlang/debian/ bionic erlang
+
+# erlang repo - aarch64 only
+#deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-bionic ./
 
 # MariaDB repo
 #deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main

docker/kolla-toolbox/Dockerfile.j2

@@ -13,10 +13,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {{ macros.configure_user(name='ansible') }}
 {{ macros.configure_user(name='rabbitmq') }}
 
-{{ macros.enable_extra_repos(['rabbitmq']) }}
+{{ macros.enable_extra_repos(['rabbitmq', 'erlang']) }}
 
-# NOTE(mandre) Remove rabbitmq-server pinning once package dependencies are met
-# https://bugs.launchpad.net/kolla/+bug/1814233
 {% if base_package_type == 'rpm' %}
     {% set kolla_toolbox_packages = [
         'crudini',
@@ -36,14 +34,12 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% if distro_package_manager == 'dnf' %}
         {% set kolla_toolbox_packages = kolla_toolbox_packages + [
             'python3-devel',
-            'rabbitmq-server-3.7.*'
+            'rabbitmq-server-3.8.*'
         ] %}
     {% else %}
-        # NOTE(mandre) Remove rabbitmq-server pinning once package dependencies are met
-        # https://bugs.launchpad.net/kolla/+bug/1814233
         {% set kolla_toolbox_packages = kolla_toolbox_packages + [
             'python-devel',
-            'rabbitmq-server-3.7.10'
+            'rabbitmq-server-3.8.*'
         ] %}
     {% endif %}
 

docker/macros.j2

@@ -87,15 +87,6 @@ RUN usermod --append --home {{ homedir }} --groups kolla {{ name }} \
 {%- endif %}
 {%- endmacro %}
 
-{% macro install_rabbitmq_plugins(plugins, version) -%}
-    ENV PLUGINS_DIR /usr/lib/rabbitmq/plugins
-{% if plugins is defined and plugins|length > 0 -%}
-    {% for item in plugins %}
-         RUN curl -L -o ${PLUGINS_DIR}/{{ item }} https://github.com/deadtrickster/prometheus_rabbitmq_exporter/releases/download/{{ version }}/{{ item }}
-    {% endfor %}
-{%- endif %}
-{%- endmacro %}
-
 {% macro enable_extra_repos(repos) %}
 
 {{ handle_repos(repos, 'enable') }}

docker/rabbitmq/Dockerfile.j2

@@ -9,44 +9,29 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {{ macros.configure_user(name='rabbitmq') }}
 
-{{ macros.enable_extra_repos(['rabbitmq']) }}
+{{ macros.enable_extra_repos(['rabbitmq', 'erlang']) }}
 
 {% block rabbitmq_install %}
 {% if base_package_type == 'rpm' %}
     {% set rabbitmq_packages = [
-        'erlang-hipe',
         'hostname',
     ] %}
     {% if distro_package_manager == 'dnf' %}
         {% set rabbitmq_packages = rabbitmq_packages + [
-            'rabbitmq-server-3.7.*'
+            'rabbitmq-server-3.8.*'
         ] %}
     {% else %}
-        # NOTE(mandre) Remove rabbitmq-server pinning once package dependencies are met
-        # https://bugs.launchpad.net/kolla/+bug/1814233
         {% set rabbitmq_packages = rabbitmq_packages + [
-            'rabbitmq-server-3.7.10'
+            'rabbitmq-server-3.8.*'
         ] %}
     {% endif %}
 
 {% elif base_package_type == 'deb' %}
     {% set rabbitmq_packages = [
-        'erlang-nox',
         'logrotate',
         'rabbitmq-server',
     ] %}
 
-    {% if base_arch == 'aarch64' %}
-         {% set rabbitmq_packages = rabbitmq_packages + [
-             'erlang-base',
-         ] %}
-    {% else %}
-         {% set rabbitmq_packages = rabbitmq_packages + [
-             'erlang-base-hipe',
-         ] %}
-
-    {% endif %}
-
 {% endif %}
 
 {{ macros.install_packages(rabbitmq_packages | customizable("packages")) }}
@@ -58,18 +43,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 RUN rm -rf /var/lib/rabbitmq/* \
     && mkdir -p /usr/lib/rabbitmq/plugins \
     && /usr/lib/rabbitmq/bin/rabbitmq-plugins enable --offline \
-       rabbitmq_management
+       rabbitmq_management \
+       rabbitmq_prometheus
 
-
-    {% set rabbitmq_plugins = [
-        'accept-0.3.3.ez',
-        'prometheus-3.5.1.ez',
-        'prometheus_cowboy-0.1.4.ez',
-        'prometheus_httpd-2.1.8.ez',
-        'prometheus_rabbitmq_exporter-3.7.2.4.ez'
-    ] %}
-
-{{ macros.install_rabbitmq_plugins(rabbitmq_plugins | customizable("plugins"), version="v3.7.2.4") }}
 {% endblock %}
 
 COPY extend_start.sh /usr/local/bin/kolla_extend_start

kolla/template/repos.yaml

@@ -4,6 +4,7 @@ centos:
   ceph: "centos-ceph-nautilus"
   elasticsearch: "elasticsearch-kibana-logstash-5.x"
   epel: "epel"
+  erlang: "rabbitmq_rabbitmq-erlang"
   extras: "extras"
   grafana: "grafana"
   hacluster: "network_ha-clustering_Stable"
@@ -20,6 +21,7 @@ centos-aarch64:
   ceph: "centos-ceph-nautilus"
   elasticsearch: "elasticsearch-kibana-logstash-5.x"
   epel: "epel"
+  erlang: "rabbitmq_rabbitmq-erlang"
   extras: "extras"
   grafana: "grafana"
   nfs-ganesha: "centos-nfs-ganesha28"
@@ -40,12 +42,13 @@ centos-ppc64le:
 # TODO(mnasiadka): Rework the repo list once Ceph Octopus is released
 centos8:
   ceph: "centos-ceph-nautilus"
-  nfs-ganesha: "centos-nfs-ganesha28"
-  epel: "epel"
   epel-modular: "epel-modular"
+  epel: "epel"
+  erlang: "rabbitmq_rabbitmq-erlang"
   grafana: "grafana"
   hacluster: "HighAvailability"
   influxdb: "influxdb"
+  nfs-ganesha: "centos-nfs-ganesha28"
   rabbitmq: "rabbitmq_rabbitmq-server"
   td-agent: "treasuredata"
 
@@ -54,38 +57,44 @@ rhel: {}
 
 debian:
   elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
   kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main"
   td-agent: "deb http://packages.treasuredata.com/3/debian/buster buster contrib"
 
 debian-aarch64:
   elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
   kibana: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./"
   libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./"
+  rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main"
 
 ubuntu:
   elasticsearch: "deb https://artifacts.elastic.co/packages/5.x/apt stable main"
+  erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ bionic erlang"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable"
   kibana: "deb https://artifacts.elastic.co/packages/5.x/apt stable main"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
   opendaylight: "deb http://ppa.launchpad.net/odl-team/carbon/ubuntu xenial main"
   qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ bionic main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main"
+  rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main"
   sensu: "deb https://sensu.global.ssl.fastly.net/apt bionic main"
   td-agent: "deb http://packages.treasuredata.com/3/ubuntu/bionic/ bionic contrib"
 
 ubuntu-aarch64:
   elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main"
+  erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-bionic ./"
   grafana: "deb https://packages.grafana.com/oss/deb stable main"
   influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable"
   kibana: "deb https://artifacts.elastic.co/packages/5.x/apt stable main"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main"
+  rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main"
 
 ubuntu-ppc64le:
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
-  rabbitmq: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main"
+  rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main"

releasenotes/notes/rabbitmq-version-upgrade-ac67bd6ae99c8169.yaml

@@ -0,0 +1,6 @@
+---
+upgrade:
+  - |
+    Upgrades RabbitMQ to 3.8. See `blueprint
+    <https://blueprints.launchpad.net/kolla/+spec/rabbitmq-version-upgrade>`__
+    for details.