Support TLS encryption of RabbitMQ client-server traffic

This change adds support for encryption of communication between
OpenStack services and RabbitMQ. Server certificates are supported, but
currently client certificates are not.

The kolla-ansible certificates command has been updated to support
generating certificates for RabbitMQ for development and testing.

RabbitMQ TLS is enabled in the all-in-one source CI jobs, or when
The Zuul 'tls_enabled' variable is true.

Change-Id: I4f1d04150fb2b5af085b762890092f87ae6076b5
Implements: blueprint message-queue-ssl-support

ansible/certificates.yml

@@ -1,6 +1,8 @@
 ---
 - import_playbook: gather-facts.yml
-  when: kolla_enable_tls_backend | default(false) | bool
+  when: >-
+    kolla_enable_tls_backend | default(false) | bool or
+    rabbitmq_enable_tls | default(false) | bool
 
 - name: Apply role certificates
   hosts: localhost

ansible/group_vars/all.yml

@@ -190,6 +190,11 @@ om_notify_vhost: "/"
 
 notify_transport_url: "{{ om_notify_transport }}://{% for host in groups[om_notify_group] %}{{ om_notify_user }}:{{ om_notify_password }}@{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ om_notify_port }}{% if not loop.last %},{% endif %}{% endfor %}/{{ om_notify_vhost }}"
 
+# Whether to enable TLS for oslo.messaging communication with RabbitMQ.
+om_enable_rabbitmq_tls: "{{ rabbitmq_enable_tls | bool }}"
+# CA certificate bundle in containers using oslo.messaging with RabbitMQ TLS.
+om_rabbitmq_cacert: "{{ rabbitmq_cacert }}"
+
 ####################
 # Networking options
 ####################
@@ -424,7 +429,7 @@ qdrouterd_port: "31459"
 
 qinling_api_port: "7070"
 
-rabbitmq_port: "5672"
+rabbitmq_port: "{{ '5671' if rabbitmq_enable_tls | bool else '5672' }}"
 rabbitmq_management_port: "15672"
 rabbitmq_cluster_port: "25672"
 rabbitmq_epmd_port: "4369"
@@ -746,6 +751,10 @@ osprofiler_backend_connection_string: "{{ redis_connection_string if osprofiler_
 rabbitmq_user: "openstack"
 rabbitmq_monitoring_user: ""
 outward_rabbitmq_user: "openstack"
+# Whether to enable TLS encryption for RabbitMQ client-server communication.
+rabbitmq_enable_tls: "no"
+# CA certificate bundle in RabbitMQ container.
+rabbitmq_cacert: "/etc/ssl/certs/{{ 'ca-certificates.crt' if kolla_base_distro in ['debian', 'ubuntu'] else 'ca-bundle.trust.crt' }}"
 
 ####################
 # Qdrouterd options

ansible/roles/aodh/templates/aodh.conf.j2

@@ -56,3 +56,8 @@ topics = {{ aodh_enabled_notification_topics | map(attribute='name') | join(',')
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}

ansible/roles/barbican/templates/barbican.conf.j2

@@ -75,6 +75,12 @@ topics = {{ barbican_enabled_notification_topics | map(attribute='name') | join(
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 [oslo_middleware]
 enable_proxy_headers_parsing = True
 

ansible/roles/blazar/templates/blazar.conf.j2

@@ -58,6 +58,11 @@ topics = {{ blazar_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
 
 {% if blazar_policy_file is defined %}
 [oslo_policy]

ansible/roles/ceilometer/templates/ceilometer.conf.j2

@@ -35,6 +35,12 @@ ca_file = /etc/ceilometer/vmware_ca
 [oslo_messaging_notifications]
 transport_url = {{ notify_transport_url }}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if ceilometer_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ ceilometer_policy_file }}

ansible/roles/certificates/tasks/generate-backend.yml

@@ -62,3 +62,16 @@
     src: "{{ backend_dir }}/backend.key"
     dest: "{{ kolla_certificates_dir }}/backend-key.pem"
     mode: "0660"
+
+- name: Copy backend TLS certificate and key for RabbitMQ
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    remote_src: true
+  with_items:
+    - src: "{{ kolla_tls_backend_cert }}"
+      dest: "{{ kolla_certificates_dir }}/rabbitmq-cert.pem"
+    - src: "{{ kolla_tls_backend_key }}"
+      dest: "{{ kolla_certificates_dir }}/rabbitmq-key.pem"
+  when:
+    - rabbitmq_enable_tls | bool

ansible/roles/certificates/tasks/main.yml

@@ -3,4 +3,4 @@
 - include_tasks: generate.yml
 - include_tasks: generate-backend.yml
   when:
-    - kolla_enable_tls_backend | bool
+    - kolla_enable_tls_backend | bool or rabbitmq_enable_tls | bool

ansible/roles/cinder/templates/cinder.conf.j2

@@ -69,6 +69,12 @@ topics = {{ cinder_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 [oslo_middleware]
 enable_proxy_headers_parsing = True
 

ansible/roles/cloudkitty/templates/cloudkitty.conf.j2

@@ -41,6 +41,12 @@ lock_path = /var/lib/cloudkitty/tmp
 policy_file = {{ cloudkitty_policy_file }}
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 [collect]
 collector = {{ cloudkitty_collector_backend }}
 {% if cloudkitty_custom_metrics_used %}

ansible/roles/cyborg/templates/cyborg.conf.j2

@@ -52,3 +52,9 @@ topics = {{ cyborg_enabled_notification_topics | map(attribute='name') | join(',
 {% else %}
 driver = noop
 {% endif %}
+
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}

ansible/roles/designate/templates/designate.conf.j2

@@ -89,6 +89,12 @@ topics = {{ designate_enabled_notification_topics | map(attribute='name') | join
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 [oslo_concurrency]
 lock_path = /var/lib/designate/tmp
 

ansible/roles/glance/templates/glance-api.conf.j2

@@ -118,6 +118,12 @@ topics = {{ glance_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if glance_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ glance_policy_file }}

ansible/roles/heat/templates/heat.conf.j2

@@ -82,6 +82,12 @@ topics = {{ heat_enabled_notification_topics | map(attribute='name') | join(',')
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if heat_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ heat_policy_file }}

ansible/roles/ironic/templates/ironic-inspector.conf.j2

@@ -12,6 +12,12 @@ transport_url = {{ rpc_transport_url }}
 [oslo_messaging_notifications]
 transport_url = {{ notify_transport_url }}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 [ironic]
 {% if enable_keystone | bool %}
 auth_url = {{ keystone_admin_url }}

ansible/roles/ironic/templates/ironic.conf.j2

@@ -30,6 +30,12 @@ topics = {{ ironic_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if ironic_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ ironic_policy_file }}

ansible/roles/karbor/templates/karbor.conf.j2

@@ -55,5 +55,11 @@ topics = {{ karbor_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/keystone/templates/keystone.conf.j2

@@ -57,6 +57,11 @@ topics = {{ keystone_enabled_notification_topics | map(attribute='name') | join(
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
 
 {% if enable_osprofiler | bool %}
 [profiler]

ansible/roles/magnum/templates/magnum.conf.j2

@@ -110,6 +110,12 @@ topics = {{ magnum_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if magnum_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ magnum_policy_file }}

ansible/roles/manila/templates/manila.conf.j2

@@ -52,6 +52,11 @@ topics = {{ manila_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/masakari/templates/masakari.conf.j2

@@ -46,6 +46,12 @@ topics = notifications
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 [oslo_middleware]
 enable_proxy_headers_parsing = True
 

ansible/roles/mistral/templates/mistral.conf.j2

@@ -68,6 +68,12 @@ topics = {{ mistral_enabled_notification_topics | map(attribute='name') | join('
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if mistral_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ mistral_policy_file }}

ansible/roles/murano/templates/murano.conf.j2

@@ -57,6 +57,12 @@ topics = {{ murano_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 [oslo_middleware]
 enable_proxy_headers_parsing = True
 

ansible/roles/neutron/templates/neutron.conf.j2

@@ -127,6 +127,12 @@ topics = {{ neutron_enabled_notification_topics | map(attribute='name') | join('
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if neutron_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ neutron_policy_file }}

ansible/roles/nova-cell/templates/nova.conf.j2

@@ -180,6 +180,12 @@ topics = {{ nova_enabled_notification_topics | map(attribute='name') | join(',')
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if service_name in nova_cell_services_require_policy_json and nova_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ nova_policy_file }}

ansible/roles/nova/templates/nova.conf.j2

@@ -129,6 +129,12 @@ topics = {{ nova_enabled_notification_topics | map(attribute='name') | join(',')
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if service_name in nova_services_require_policy_json and nova_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ nova_policy_file }}

ansible/roles/octavia/templates/octavia.conf.j2

@@ -86,6 +86,12 @@ rpc_thread_pool_size = 2
 [oslo_messaging_notifications]
 transport_url = {{ notify_transport_url }}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if octavia_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ octavia_policy_file }}

ansible/roles/qinling/templates/qinling.conf.j2

@@ -53,6 +53,12 @@ topics = notifications
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if qinling_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ qinling_policy_file }}

ansible/roles/rabbitmq/defaults/main.yml

@@ -72,3 +72,5 @@ rabbitmq_cluster_name: "openstack"
 rabbitmq_hostname: "{{ ansible_hostname }}"
 rabbitmq_pid_file: "/var/lib/rabbitmq/mnesia/rabbitmq.pid"
 rabbitmq_server_additional_erl_args: ""
+# Dict of TLS options for RabbitMQ. Keys will be prefixed with 'ssl_options.'.
+rabbitmq_tls_options: {}

ansible/roles/rabbitmq/handlers/main.yml

@@ -17,6 +17,7 @@
     - inventory_hostname == groups[service.group]|first
   notify:
     - Waiting for rabbitmq to start on first node
+  listen: Restart rabbitmq container
 
 - name: Waiting for rabbitmq to start on first node
   vars:
@@ -43,3 +44,4 @@
   when:
     - kolla_action != "config"
     - inventory_hostname != groups[service.group]|first
+  listen: Restart rabbitmq container

ansible/roles/rabbitmq/tasks/check-containers.yml

@@ -14,5 +14,4 @@
     - item.value.enabled | bool
   with_dict: "{{ rabbitmq_services }}"
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container

ansible/roles/rabbitmq/tasks/config.yml

@@ -23,8 +23,7 @@
     - item.value.enabled | bool
   with_dict: "{{ rabbitmq_services }}"
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container
 
 - name: Copying over rabbitmq-env.conf
   become: true
@@ -42,9 +41,7 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
-
+    - Restart rabbitmq container
 
 - name: Copying over rabbitmq.conf
   become: true
@@ -62,8 +59,7 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container
 
 - name: Copying over erl_inetrc
   become: true
@@ -81,8 +77,7 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container
 
 - name: Copying over definitions.json
   become: true
@@ -100,8 +95,10 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container
+
+- include_tasks: copy-certs.yml
+  when: rabbitmq_enable_tls | bool
 
 - import_tasks: check-containers.yml
   when: kolla_action != "config"

ansible/roles/rabbitmq/tasks/copy-certs.yml

@@ -0,0 +1,52 @@
+---
+- name: Copying over extra CA certificates
+  become: true
+  vars:
+    service: "{{ rabbitmq_services['rabbitmq'] }}"
+  copy:
+    src: "{{ kolla_certificates_dir }}/ca/"
+    dest: "{{ node_config_directory }}/{{ project_name }}/ca-certificates"
+    mode: "0644"
+  when:
+    - kolla_copy_ca_into_containers | bool
+    - service | service_enabled_and_mapped_to_host
+  notify:
+    - Restart rabbitmq container
+
+- name: Copying over TLS certificate
+  become: true
+  vars:
+    service: "{{ rabbitmq_services['rabbitmq'] }}"
+  copy:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/{{ project_name }}/{{ project_name }}-cert.pem"
+    mode: "0644"
+  with_first_found:
+    - files:
+        - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/{{ project_name }}-cert.pem"
+        - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-cert.pem"
+        - "{{ kolla_certificates_dir }}/{{ project_name }}-cert.pem"
+      skip: true
+  when:
+    - service | service_enabled_and_mapped_to_host
+  notify:
+    - Restart rabbitmq container
+
+- name: Copying over TLS key
+  become: true
+  vars:
+    service: "{{ rabbitmq_services['rabbitmq'] }}"
+  copy:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/{{ project_name }}/{{ project_name }}-key.pem"
+    mode: "0600"
+  with_first_found:
+    - files:
+        - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/{{ project_name }}-key.pem"
+        - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-key.pem"
+        - "{{ kolla_certificates_dir }}/{{ project_name }}-key.pem"
+      skip: true
+  when:
+    - service | service_enabled_and_mapped_to_host
+  notify:
+    - Restart rabbitmq container

ansible/roles/rabbitmq/tasks/precheck.yml

@@ -74,6 +74,32 @@
   when:
     - not item.1 is match('^'+('api' | kolla_address(item.0.item))+'\\b')
 
+- name: Check if TLS certificate exists for RabbitMQ
+  vars:
+    cert: "{{ query('first_found', paths, errors='ignore') }}"
+    paths:
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/rabbitmq-cert.pem"
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-cert.pem"
+      - "{{ kolla_certificates_dir }}/rabbitmq-cert.pem"
+  fail:
+    msg: No TLS certificate provided for RabbitMQ.
+  when:
+    - rabbitmq_enable_tls | bool
+    - cert | length == 0
+
+- name: Check if TLS key exists for RabbitMQ
+  vars:
+    key: "{{ query('first_found', paths, errors='ignore') }}"
+    paths:
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/rabbitmq-key.pem"
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-key.pem"
+      - "{{ kolla_certificates_dir }}/rabbitmq-key.pem"
+  fail:
+    msg: No TLS key provided for RabbitMQ.
+  when:
+    - rabbitmq_enable_tls | bool
+    - key | length == 0
+
 - name: Checking free port for outward RabbitMQ
   wait_for:
     host: "{{ api_interface_address }}"
@@ -137,3 +163,31 @@
   when:
     - enable_outward_rabbitmq | bool
     - not item.1 is match('^'+('api' | kolla_address(item.0.item))+'\\b')
+
+- name: Check if TLS certificate exists for outward RabbitMQ
+  vars:
+    cert: "{{ query('first_found', paths, errors='ignore') }}"
+    paths:
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/outward_rabbitmq-cert.pem"
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-cert.pem"
+      - "{{ kolla_certificates_dir }}/outward_rabbitmq-cert.pem"
+  fail:
+    msg: No TLS certificate provided for outward RabbitMQ.
+  when:
+    - enable_outward_rabbitmq | bool
+    - rabbitmq_enable_tls | bool
+    - cert | length == 0
+
+- name: Check if TLS key exists for outward RabbitMQ
+  vars:
+    key: "{{ query('first_found', paths, errors='ignore') }}"
+    paths:
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/outward_rabbitmq-key.pem"
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-key.pem"
+      - "{{ kolla_certificates_dir }}/outward_rabbitmq-key.pem"
+  fail:
+    msg: No TLS key provided for outward RabbitMQ.
+  when:
+    - enable_outward_rabbitmq | bool
+    - rabbitmq_enable_tls | bool
+    - key | length == 0

ansible/roles/rabbitmq/templates/rabbitmq.conf.j2

@@ -1,6 +1,11 @@
 # NOTE(yoctozepto): rabbitmq uses the raw format (e.g. fd::) of IPv6 address;
 # despite specifying port via colon, the url format (e.g. [fd::]) is not accepted
+{% if rabbitmq_enable_tls | bool %}
+listeners.tcp = none
+listeners.ssl.1 = {{ api_interface_address }}:{{ role_rabbitmq_port }}
+{% else %}
 listeners.tcp.1 = {{ api_interface_address }}:{{ role_rabbitmq_port }}
+{% endif %}
 {# NOTE: to avoid split-brain #}
 cluster_partition_handling = pause_minority
 
@@ -12,3 +17,12 @@ cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config
 {% for host in groups[role_rabbitmq_groups] %}
 cluster_formation.classic_config.nodes.{{ loop.index0 }} = rabbit@{{ hostvars[host]['ansible_hostname'] }}
 {% endfor %}
+
+{% if rabbitmq_enable_tls | bool %}
+# https://www.rabbitmq.com/ssl.html
+ssl_options.certfile = /etc/rabbitmq/certs/{{ project_name }}-cert.pem
+ssl_options.keyfile = /etc/rabbitmq/certs/{{ project_name }}-key.pem
+{% for key, value in rabbitmq_tls_options.items() %}
+ssl_options.{{ key }} = {{ value }}
+{% endfor %}
+{% endif %}

ansible/roles/rabbitmq/templates/rabbitmq.json.j2

@@ -24,7 +24,19 @@
             "dest": "/etc/rabbitmq/definitions.json",
             "owner": "rabbitmq",
             "perm": "0600"
-        }
+        }{% if rabbitmq_enable_tls | bool %},
+        {
+            "source": "{{ container_config_directory }}/{{ project_name }}-cert.pem",
+            "dest": "/etc/rabbitmq/certs/{{ project_name }}-cert.pem",
+            "owner": "rabbitmq",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/{{ project_name }}-key.pem",
+            "dest": "/etc/rabbitmq/certs/{{ project_name }}-key.pem",
+            "owner": "rabbitmq",
+            "perm": "0600"
+        }{% endif %}
     ],
     "permissions": [
         {

ansible/roles/sahara/templates/sahara.conf.j2

@@ -36,6 +36,12 @@ topics = {{ sahara_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if sahara_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ sahara_policy_file }}

ansible/roles/searchlight/templates/searchlight.conf.j2

@@ -44,6 +44,11 @@ topics = {{ searchlight_enabled_notification_topics | map(attribute='name') | jo
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
 
 {% if searchlight_policy_file is defined %}
 [oslo_policy]

ansible/roles/senlin/templates/senlin.conf.j2

@@ -64,6 +64,12 @@ topics = {{ senlin_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if senlin_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ senlin_policy_file }}

ansible/roles/solum/templates/solum.conf.j2

@@ -59,3 +59,9 @@ memcached_servers = {% for host in groups['memcached'] %}{{ 'api' | kolla_addres
 
 [oslo_messaging_notifications]
 transport_url = {{ notify_transport_url }}
+
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}

ansible/roles/tacker/templates/tacker.conf.j2

@@ -64,6 +64,12 @@ topics = {{ tacker_enabled_notification_topics | map(attribute='name') | join(',
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if tacker_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ tacker_policy_file }}

ansible/roles/trove/templates/trove-conductor.conf.j2

@@ -25,6 +25,12 @@ topics = {{ trove_enabled_notification_topics | map(attribute='name') | join(','
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 [database]
 connection = mysql+pymysql://{{ trove_database_user }}:{{ trove_database_password }}@{{ trove_database_address }}/{{ trove_database_name }}
 max_retries = -1

ansible/roles/trove/templates/trove-taskmanager.conf.j2

@@ -46,6 +46,11 @@ topics = {{ trove_enabled_notification_topics | map(attribute='name') | join(','
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
 
 {% if enable_osprofiler | bool %}
 [profiler]

ansible/roles/trove/templates/trove.conf.j2

@@ -53,6 +53,12 @@ topics = {{ trove_enabled_notification_topics | map(attribute='name') | join(','
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if enable_osprofiler | bool %}
 [profiler]
 enabled = true

ansible/roles/vitrage/templates/vitrage.conf.j2

@@ -70,6 +70,12 @@ topics = {{ vitrage_enabled_notification_topics | map(attribute='name') | join('
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 [oslo_concurrency]
 lock_path = /var/lib/vitrage/tmp
 

ansible/roles/watcher/templates/watcher.conf.j2

@@ -55,6 +55,12 @@ topics = {{ watcher_enabled_notification_topics | map(attribute='name') | join('
 driver = noop
 {% endif %}
 
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}
+
 {% if watcher_policy_file is defined %}
 [oslo_policy]
 policy_file = {{ watcher_policy_file }}

ansible/roles/zun/templates/zun.conf.j2

@@ -125,3 +125,9 @@ docker_remote_api_port = 2375
 
 [cni_daemon]
 cni_daemon_port = {{ zun_cni_daemon_port }}
+
+{% if om_enable_rabbitmq_tls | bool %}
+[oslo_messaging_rabbit]
+ssl = true
+ssl_ca_file = {{ om_rabbitmq_cacert }}
+{% endif %}

doc/source/admin/advanced-configuration.rst

@@ -69,6 +69,8 @@ RabbitMQ doesn't work with IP address, hence the IP address of
 ``api_interface`` should be resolvable by hostnames to make sure that
 all RabbitMQ Cluster hosts can resolve each others hostname beforehand.
 
+.. _tls-configuration:
+
 TLS Configuration
 ~~~~~~~~~~~~~~~~~
 

doc/source/conf.py

@@ -98,6 +98,7 @@ openstack_projects = [
     'neutron',
     'nova',
     'octavia',
+    'oslo.messaging',
     'oslotest',
     'swift',
 ]

doc/source/reference/message-queues/rabbitmq.rst

@@ -8,6 +8,81 @@ RabbitMQ is a message broker written in Erlang.
 It is currently the default provider of message queues in Kolla Ansible
 deployments.
 
+TLS encryption
+~~~~~~~~~~~~~~
+
+There are a number of channels to consider when securing RabbitMQ
+communication. Kolla Ansible currently supports TLS encryption of the
+following:
+
+* client-server traffic, typically between OpenStack services using the
+  :oslo.messaging-doc:`oslo.messaging </>` library and RabbitMQ
+* RabbitMQ Management API and UI (frontend connection to HAProxy only)
+
+Encryption of the following channels is not currently supported:
+
+* RabbitMQ cluster traffic between RabbitMQ server nodes
+* RabbitMQ CLI communication with RabbitMQ server nodes
+* RabbitMQ Management API and UI (backend connection from HAProxy to RabbitMQ)
+
+Client-server
+-------------
+
+Encryption of client-server traffic is enabled by setting
+``rabbitmq_enable_tls`` to ``true``. Additionally, certificates and keys must
+be available in the following paths (in priority order):
+
+Certificates:
+
+* ``"{{ kolla_certificates_dir }}/{{ inventory_hostname }}/rabbitmq-cert.pem"``
+* ``"{{ kolla_certificates_dir }}/{{ inventory_hostname }}-cert.pem"``
+* ``"{{ kolla_certificates_dir }}/rabbitmq-cert.pem"``
+
+Keys:
+
+* ``"{{ kolla_certificates_dir }}/{{ inventory_hostname }}/rabbitmq-key.pem"``
+* ``"{{ kolla_certificates_dir }}/{{ inventory_hostname }}-key.pem"``
+* ``"{{ kolla_certificates_dir }}/rabbitmq-key.pem"``
+
+The default for ``kolla_certificates_dir`` is ``/etc/kolla/certificates``.
+
+The certificates must be valid for the IP address of the host running RabbitMQ
+on the API network.
+
+Additional TLS configuration options may be passed to RabbitMQ via
+``rabbitmq_tls_options``. This should be a dict, and the keys will be prefixed
+with ``ssl_options.``. For example:
+
+.. code-block:: yaml
+
+   rabbitmq_tls_options:
+     ciphers.1: ECDHE-ECDSA-AES256-GCM-SHA384
+     ciphers.2: ECDHE-RSA-AES256-GCM-SHA384
+     ciphers.3: ECDHE-ECDSA-AES256-SHA384
+     honor_cipher_order: true
+     honor_ecc_order: true
+
+Details on configuration of RabbitMQ for TLS can be found in the `RabbitMQ
+documentation <https://www.rabbitmq.com/ssl.html>`__.
+
+When ``om_rabbitmq_enable_tls`` is ``true`` (it defaults to the value of
+``rabbitmq_enable_tls``), applicable OpenStack services will be configured to
+use oslo.messaging with TLS enabled. The CA certificate is configured via
+``om_rabbitmq_cacert`` (it defaults to ``rabbitmq_cacert``, which points to the
+system's trusted CA certificate bundle for TLS). Note that there is currently
+no support for using client certificates.
+
+For testing purposes, Kolla Ansible provides the ``kolla-ansible certificates``
+command, which will generate self-signed certificates for RabbitMQ if
+``rabbitmq_enable_tls`` is ``true``.
+
+Management API and UI
+---------------------
+
+The management API and UI are accessed via HAProxy, exposed only on the
+internal VIP. As such, traffic to this endpoint is encrypted when
+``kolla_enable_tls_internal`` is ``true``. See :ref:`tls-configuration`.
+
 Passing arguments to RabbitMQ server's Erlang VM
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

etc/kolla/globals.yml

@@ -84,6 +84,10 @@
 #om_rpc_port: "{{ qdrouterd_port }}"
 #om_rpc_group: "qdrouterd"
 
+# Whether to enable TLS for oslo.messaging communication with RabbitMQ.
+#om_enable_rabbitmq_tls: "{{ rabbitmq_enable_tls | bool }}"
+# CA certificate bundle in containers using oslo.messaging with RabbitMQ TLS.
+#om_rabbitmq_cacert: "{{ rabbitmq_cacert }}"
 
 ##############################
 # Neutron - Networking Options
@@ -373,6 +377,10 @@
 # These are appended to args already provided by Kolla Ansible
 # to configure IPv6 in RabbitMQ server.
 #rabbitmq_server_additional_erl_args: ""
+# Whether to enable TLS encryption for RabbitMQ client-server communication.
+#rabbitmq_enable_tls: "no"
+# CA certificate bundle in RabbitMQ container.
+#rabbitmq_cacert: "/etc/ssl/certs/{{ 'ca-certificates.crt' if kolla_base_distro in ['debian', 'ubuntu'] else 'ca-bundle.trust.crt' }}"
 
 #################
 # MariaDB options

releasenotes/notes/rabbitmq-tls-78ea3fddf67267f2.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds support for TLS encryption of RabbitMQ client-server communication.
+    See `blueprint
+    <https://blueprints.launchpad.net/kolla-ansible/+spec/message-queue-ssl-support>`__
+    for details.

tests/templates/globals-default.j2

@@ -134,6 +134,7 @@ openstack_cacert: "/etc/ssl/certs/ca-certificates.crt"
 openstack_cacert: "/etc/pki/tls/certs/ca-bundle.crt"
 {% endif %}
 kolla_admin_openrc_cacert: "{% raw %}{{ kolla_certificates_dir }}{% endraw %}/ca/root.crt"
+rabbitmq_enable_tls: "yes"
 {% endif %}
 
 {% if scenario == 'linuxbridge' %}