Add a new 'outward' rabbitmq instance

Certain services such as Murano and trove require access to a rabbitmq
instance from tenant networks. [0]

Exposing the internal rabbitmq to end users is a security hole, hence
there are two options, 1) use vhosts in the existing rabbitmq, or two a
separate rabbitmq instances. Given the importance of rabbitmq to the
OpenStack deployment, we have decided to go with a separate instance.
Refer to [1] for more detail on the various options.

This change makes the rabbitmq role generic so that it can be reused, in
this case to start 'outward_rabbitmq'. It needs to be exposed via
haproxy both for network isolation and also because this is what Murano
configuration requires.

Follow on patches will be added to add a vhost in this outward instance
for Murano and other services which require access.

Based on the original work by bdaca[2]

[0] http://murano.readthedocs.io/en/stable-liberty/intro/architecture.html
[1] http://lists.openstack.org/pipermail/openstack-dev/2016-December/109091.html
[2] https://review.openstack.org/#/c/374525

Change-Id: Ib2bcc7ed4bf4f883a7cd1dfad3db89201e3cfd8d
Partial-Bug: #1620374
Depends-On: I020eb6219f89a310451becde41f6f1c7f54baadd
Co-Authored-By: Bartłomiej Daca <bartek.daca@gmail.com>

ansible/group_vars/all.yml

@@ -162,6 +162,11 @@ rabbitmq_management_port: "15672"
 rabbitmq_cluster_port: "25672"
 rabbitmq_epmd_port: "4369"
 
+outward_rabbitmq_port: "5674"
+outward_rabbitmq_management_port: "15674"
+outward_rabbitmq_cluster_port: "25674"
+outward_rabbitmq_epmd_port: "4371"
+
 mongodb_port: "27017"
 mongodb_web_port: "28017"
 
@@ -300,6 +305,7 @@ enable_memcached: "yes"
 enable_neutron: "yes"
 enable_nova: "yes"
 enable_rabbitmq: "yes"
+enable_outward_rabbitmq: "{{ enable_murano | bool }}"
 
 # Additional optional OpenStack features and services are specified here
 enable_aodh: "no"
@@ -411,6 +417,7 @@ enable_kibana: "{{ 'yes' if enable_central_logging | bool else 'no' }}"
 ####################
 rabbitmq_user: "openstack"
 rabbitmq_version: "rabbitmq_server-3.6/plugins/rabbitmq_clusterer-3.6.x.ez/rabbitmq_clusterer-3.6.x-667f92b0/ebin"
+outward_rabbitmq_user: "openstack"
 
 ####################
 # HAProxy options
@@ -520,6 +527,7 @@ nova_backend: "{{ 'rbd' if nova_backend_ceph | bool else 'default' }}"
 #######################
 horizon_backend_database: "{{ enable_murano | bool }}"
 
+
 #################
 # Octavia options
 #################

ansible/inventory/all-in-one

@@ -66,6 +66,9 @@ control
 [rabbitmq:children]
 control
 
+[outward-rabbitmq:children]
+control
+
 [mongodb:children]
 control
 

ansible/inventory/multinode

@@ -87,6 +87,9 @@ control
 [rabbitmq:children]
 control
 
+[outward-rabbitmq:children]
+control
+
 [mongodb:children]
 control
 

ansible/roles/common/tasks/config.yml

@@ -128,6 +128,7 @@
     - { name: "neutron", enabled: "{{ enable_neutron }}" }
     - { name: "nova", enabled: "{{ enable_nova }}" }
     - { name: "octavia", enabled: "{{ enable_octavia }}" }
+    - { name: "outward-rabbitmq", enabled: "{{ enable_outward_rabbitmq }}" }
     - { name: "panko", enabled: "{{ enable_panko }}" }
     - { name: "rabbitmq", enabled: "{{ enable_rabbitmq }}" }
     - { name: "rally", enabled: "{{ enable_rally }}" }

ansible/roles/common/templates/cron-logrotate-outward-rabbitmq.conf.j2

@@ -0,0 +1,3 @@
+"/var/log/kolla/outward-rabbitmq/*.log"
+{
+}

ansible/roles/common/templates/cron.json.j2

@@ -36,6 +36,7 @@
     ( 'neutron', enable_neutron ),
     ( 'nova', enable_nova ),
     ( 'octavia', enable_octavia ),
+    ( 'outward-rabbitmq', enable_outward_rabbitmq ),
     ( 'panko', enable_panko ),
     ( 'rabbitmq', enable_rabbitmq ),
     ( 'rally', enable_rally ),

ansible/roles/haproxy/tasks/precheck.yml

@@ -564,6 +564,17 @@
     - inventory_hostname in groups['haproxy']
     - haproxy_stat.find('rabbitmq_management') == -1
 
+- name: Checking free port for outward RabbitMQ Management HAProxy
+  wait_for:
+    host: "{{ kolla_internal_vip_address }}"
+    port: "{{ outward_rabbitmq_management_port }}"
+    connect_timeout: 1
+    state: stopped
+  when:
+    - enable_outward_rabbitmq | bool
+    - inventory_hostname in groups['haproxy']
+    - haproxy_stat.find('outward_rabbitmq_management') == -1
+
 - name: Checking free port for RadosGW HAProxy
   wait_for:
     host: "{{ kolla_internal_vip_address }}"

ansible/roles/haproxy/templates/haproxy.cfg.j2

@@ -44,6 +44,24 @@ listen rabbitmq_management
 {% for host in groups['rabbitmq'] %}
   server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ rabbitmq_management_port }} check inter 2000 rise 2 fall 5
 {% endfor %}
+
+listen outward_rabbitmq_management
+  bind {{ kolla_internal_vip_address }}:{{ outward_rabbitmq_management_port }}
+{% for host in groups['outward-rabbitmq'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ outward_rabbitmq_management_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% if haproxy_enable_external_vip | bool %}
+
+listen outward_rabbitmq_external
+  mode tcp
+  option tcplog
+  timeout client 3600s
+  timeout server 3600s
+  bind {{ kolla_external_vip_address }}:{{ outward_rabbitmq_port }}
+{% for host in groups['outward-rabbitmq'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ outward_rabbitmq_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
 {% endif %}
 
 {% if enable_mongodb | bool %}

ansible/roles/rabbitmq/defaults/main.yml

@@ -3,14 +3,18 @@ project_name: "rabbitmq"
 
 rabbitmq_services:
   rabbitmq:
-    container_name: rabbitmq
-    group: rabbitmq
+    container_name: "{{ project_name }}"
+    group: "{{ role_rabbitmq_groups }}"
     enabled: true
     image: "{{ rabbitmq_image_full }}"
+    environment:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      RABBITMQ_CLUSTER_COOKIE: "{{ role_rabbitmq_cluster_cookie }}"
+      RABBITMQ_LOG_DIR: "/var/log/kolla/{{ project_name }}"
     volumes:
-      - "{{ node_config_directory }}/rabbitmq/:{{ container_config_directory }}/:ro"
+      - "{{ node_config_directory }}/{{ project_name }}/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
-      - "rabbitmq:/var/lib/rabbitmq/"
+      - "{{ project_name }}:/var/lib/rabbitmq/"
       - "kolla_logs:/var/log/kolla/"
 
 

ansible/roles/rabbitmq/handlers/main.yml

@@ -11,6 +11,7 @@
     name: "{{ service.container_name }}"
     image: "{{ service.image }}"
     volumes: "{{ service.volumes }}"
+    environment: "{{ service.environment }}"
   when:
     - action != "config"
     - inventory_hostname in groups[service.group]

ansible/roles/rabbitmq/tasks/bootstrap.yml

@@ -3,7 +3,7 @@
   kolla_docker:
     action: "create_volume"
     common_options: "{{ docker_common_options }}"
-    name: "rabbitmq"
+    name: "{{ project_name }}"
   register: rabbitmq_volume
 
 - name: Running RabbitMQ bootstrap container
@@ -14,14 +14,15 @@
     environment:
       KOLLA_BOOTSTRAP:
       KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-      RABBITMQ_CLUSTER_COOKIE: "{{ rabbitmq_cluster_cookie }}"
+      RABBITMQ_CLUSTER_COOKIE: "{{ role_rabbitmq_cluster_cookie }}"
+      RABBITMQ_LOG_DIR: "/var/log/kolla/{{ project_name }}"
     image: "{{ rabbitmq_image_full }}"
     labels:
       BOOTSTRAP:
-    name: "rabbitmq_bootstrap"
+    name: "{{ project_name }}_bootstrap"
     restart_policy: "never"
     volumes:
-      - "{{ node_config_directory }}/rabbitmq/:{{ container_config_directory }}/:ro"
+      - "{{ node_config_directory }}/{{ project_name }}/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
-      - "rabbitmq:/var/lib/rabbitmq/"
+      - "{{ project_name }}:/var/lib/rabbitmq/"
   when: rabbitmq_volume | changed

ansible/roles/rabbitmq/tasks/config.yml

@@ -1,7 +1,7 @@
 ---
 - name: Ensuring config directories exist
   file:
-    path: "{{ node_config_directory }}/{{ item.key }}"
+    path: "{{ node_config_directory }}/{{ project_name }}"
     state: "directory"
     recurse: yes
   when:
@@ -12,7 +12,7 @@
 - name: Copying over config.json files for services
   template:
     src: "{{ item.key }}.json.j2"
-    dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
+    dest: "{{ node_config_directory }}/{{ project_name }}/config.json"
   register: rabbitmq_config_jsons
   when:
     - inventory_hostname in groups[item.value.group]
@@ -26,7 +26,7 @@
     service: "{{ rabbitmq_services['rabbitmq'] }}"
   template:
     src: "{{ item }}.j2"
-    dest: "{{ node_config_directory }}/rabbitmq/{{ item }}"
+    dest: "{{ node_config_directory }}/{{ project_name }}/{{ item }}"
   register: rabbitmq_confs
   when:
     - inventory_hostname in groups[service.group]
@@ -46,6 +46,7 @@
     name: "{{ item.value.container_name }}"
     image: "{{ item.value.image }}"
     volumes: "{{ item.value.volumes }}"
+    environment: "{{ item.value.environment }}"
   register: check_rabbitmq_containers
   when:
     - action != "config"

ansible/roles/rabbitmq/tasks/main.yml

@@ -1,3 +1,3 @@
 ---
 - include: "{{ action }}.yml"
-  when: inventory_hostname in groups['rabbitmq']
+  when: inventory_hostname in groups[role_rabbitmq_groups]

ansible/roles/rabbitmq/tasks/precheck.yml

@@ -59,3 +59,57 @@
   with_items: "{{ rabbitmq_hostnames.results }}"
   when:
     - "item.stdout.find(hostvars[item['item']]['ansible_' ~ hostvars[item['item']]['api_interface']]['ipv4']['address']) == -1"
+
+- name: Checking free port for outward RabbitMQ
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ outward_rabbitmq_port }}"
+    connect_timeout: 1
+    state: stopped
+  when:
+    - enable_outward_rabbitmq | bool
+    - inventory_hostname in groups['outward-rabbitmq']
+
+- name: Checking free port for outward RabbitMQ Management
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ outward_rabbitmq_management_port }}"
+    connect_timeout: 1
+    state: stopped
+  when:
+    - enable_outward_rabbitmq | bool
+    - inventory_hostname in groups['outward-rabbitmq']
+
+- name: Checking free port for outward RabbitMQ Cluster
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ outward_rabbitmq_cluster_port }}"
+    connect_timeout: 1
+    state: stopped
+  when:
+    - enable_outward_rabbitmq | bool
+    - inventory_hostname in groups['outward-rabbitmq']
+
+- name: Checking free port for outward RabbitMQ EPMD
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ outward_rabbitmq_epmd_port }}"
+    connect_timeout: 1
+    state: stopped
+  when:
+    - enable_outward_rabbitmq | bool
+    - inventory_hostname in groups['outward-rabbitmq']
+
+- name: Check if all outward rabbit hostnames are resolvable
+  command: "getent ahostsv4 {{ hostvars[item]['ansible_hostname'] }}"
+  changed_when: false
+  register: outward_rabbitmq_hostnames
+  with_items: "{{ groups['outward-rabbitmq'] }}"
+  when:
+    - enable_outward_rabbitmq | bool
+
+- fail: msg="Hostname has to resolve to IP address of api_interface"
+  with_items: "{{ outward_rabbitmq_hostnames.results }}"
+  when:
+    - enable_outward_rabbitmq | bool
+    - "item.stdout.find(hostvars[item['item']]['ansible_' ~ hostvars[item['item']]['api_interface']]['ipv4']['address']) == -1"

ansible/roles/rabbitmq/tasks/upgrade.yml

@@ -1,17 +1,21 @@
 ---
 - name: Checking if rabbitmq container needs upgrading
+  vars:
+    service_name: "rabbitmq"
+    service: "{{ rabbitmq_services[service_name] }}"
   kolla_docker:
     action: "compare_image"
     common_options: "{{ docker_common_options }}"
-    name: "rabbitmq"
+    name: "{{ project_name }}"
     image: "{{ rabbitmq_image_full }}"
-  when: inventory_hostname in groups['rabbitmq']
+    environment: "{{ service.environment }}"
+  when: inventory_hostname in groups[role_rabbitmq_groups]
   register: rabbitmq_differs
 
 - include: config.yml
 
 - name: Find gospel node
-  command: docker exec -t rabbitmq /usr/local/bin/rabbitmq_get_gospel_node
+  command: docker exec -t {{ project_name }} /usr/local/bin/rabbitmq_get_gospel_node
   changed_when: "{{ (gospel_node.stdout | from_json).changed }}"
   failed_when: "{{ (gospel_node.stdout | from_json).failed }}"
   register: gospel_node
@@ -21,7 +25,7 @@
   kolla_docker:
     action: "stop_container"
     common_options: "{{ docker_common_options }}"
-    name: "rabbitmq"
+    name: "{{ project_name }}"
   when:
     - rabbitmq_hostname != (gospel_node.stdout | from_json).hostname
     - rabbitmq_differs['result']

ansible/roles/rabbitmq/templates/definitions.json.j2

@@ -3,10 +3,10 @@
     {"name": "/"}
   ],
   "users": [
-    {"name": "{{ rabbitmq_user }}", "password": "{{ rabbitmq_password }}", "tags": "administrator"}
+    {"name": "{{ role_rabbitmq_user }}", "password": "{{ role_rabbitmq_password }}", "tags": "administrator"}
   ],
   "permissions": [
-    {"user": "{{ rabbitmq_user }}", "vhost": "/", "configure": ".*", "write": ".*", "read": ".*"}
+    {"user": "{{ role_rabbitmq_user }}", "vhost": "/", "configure": ".*", "write": ".*", "read": ".*"}
   ],
   "policies":[
     {"vhost": "/", "name": "ha-all", "pattern": ".*", "apply-to": "all", "definition": {"ha-mode":"all"}, "priority":0}

ansible/roles/rabbitmq/templates/rabbitmq-clusterer.config.j2

@@ -1,11 +1,11 @@
 [
   {version, 1},
   {nodes, [
-  {% for host in groups['rabbitmq'] %}
+  {% for host in groups[role_rabbitmq_groups] %}
     {'rabbit@{{ hostvars[host]['ansible_hostname'] }}', disc}
       {%- if not loop.last -%},{%- endif %}
   {% endfor %}
   ]},
   {gospel,
-    {node, 'rabbit@{{ hostvars[groups['rabbitmq'][0]]['ansible_hostname'] }}'}}
+    {node, 'rabbit@{{ hostvars[groups[role_rabbitmq_groups][0]]['ansible_hostname'] }}'}}
 ].

ansible/roles/rabbitmq/templates/rabbitmq-env.conf.j2

@@ -4,7 +4,7 @@ RABBITMQ_BOOT_MODULE=rabbit_clusterer
 RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-pa /usr/lib/rabbitmq/lib/{{ rabbitmq_version }}"
 {%- endif %}
 
-RABBITMQ_LOG_BASE=/var/log/kolla/rabbitmq
+RABBITMQ_LOG_BASE=/var/log/kolla/{{ project_name }}
 
 # TODO(sdake, vhosakot)
 # erlang by default binds to wildcard (all interfaces) and can potentially
@@ -16,4 +16,4 @@ RABBITMQ_LOG_BASE=/var/log/kolla/rabbitmq
 # https://bugs.launchpad.net/kolla/+bug/1562701
 # https://bugzilla.redhat.com/show_bug.cgi?id=1324922
 #export ERL_EPMD_ADDRESS={{ api_interface_address }}
-export ERL_EPMD_PORT={{ rabbitmq_epmd_port }}
+export ERL_EPMD_PORT={{ role_rabbitmq_epmd_port }}

ansible/roles/rabbitmq/templates/rabbitmq.config.j2

@@ -1,12 +1,12 @@
 [
   {kernel, [
     {inet_dist_use_interface, {% raw %}{{% endraw %}{{ api_interface_address | regex_replace('\.', ',') }}}},
-    {inet_dist_listen_min, {{ rabbitmq_cluster_port }}},
-    {inet_dist_listen_max, {{ rabbitmq_cluster_port }}}
+    {inet_dist_listen_min, {{ role_rabbitmq_cluster_port }}},
+    {inet_dist_listen_max, {{ role_rabbitmq_cluster_port }}}
   ]},
   {rabbit, [
     {tcp_listeners, [
-      {"{{ api_interface_address }}", {{ rabbitmq_port }}}
+      {"{{ api_interface_address }}", {{ role_rabbitmq_port }}}
     ]}{% if orchestration_engine == 'ANSIBLE' %},
     {cluster_partition_handling, autoheal}
     {%- endif %}
@@ -14,7 +14,7 @@
   {rabbitmq_management, [
     {listener, [
       {ip, "{{ api_interface_address }}"},
-      {port, {{ rabbitmq_management_port }}}
+      {port, {{ role_rabbitmq_management_port }}}
     ]},
     {load_definitions, "/etc/rabbitmq/definitions.json"}
   ]}{% if orchestration_engine == 'ANSIBLE' %},

ansible/roles/telegraf/defaults/main.yml

@@ -33,3 +33,4 @@ haproxy_proto: "http"
 influxdb_proto: "http"
 rabbitmq_proto: "http"
 mariadb_proto: "tcp"
+outward_rabbitmq_proto: "http"

ansible/roles/telegraf/templates/telegraf.conf.j2

@@ -53,6 +53,12 @@
   username = "{{ rabbitmq_user }}"
   password = "{{ rabbitmq_password }}"
 {% endif %}
+{% if inventory_hostname in groups['outward-rabbitmq'] and enable_outward_rabbitmq | bool %}
+[[inputs.rabbitmq]]
+  url = "{{ outward_rabbitmq_proto }}://{{ api_interface_address }}:{{ outward_rabbitmq_management_port }}"
+  username = "{{ outward_rabbitmq_user }}"
+  password = "{{ outward_rabbitmq_password }}"
+{% endif %}
 {% if inventory_hostname in groups['mariadb'] and enable_mariadb | bool %}
 [[inputs.mysql]]
   servers = ["{{ database_user }}:{{ database_password }}@{{ mariadb_proto }}({{ api_interface_address }}:{{ database_port }})/"]

ansible/site.yml

@@ -165,8 +165,33 @@
   roles:
     - { role: rabbitmq,
         tags: rabbitmq,
+        role_rabbitmq_cluster_cookie: '{{ rabbitmq_cluster_cookie }}',
+        role_rabbitmq_cluster_port: '{{ rabbitmq_cluster_port }}',
+        role_rabbitmq_epmd_port: '{{ rabbitmq_epmd_port }}',
+        role_rabbitmq_groups: rabbitmq,
+        role_rabbitmq_management_port: '{{ rabbitmq_management_port }}',
+        role_rabbitmq_password: '{{ rabbitmq_password }}',
+        role_rabbitmq_port: '{{ rabbitmq_port }}',
+        role_rabbitmq_user: '{{ rabbitmq_user }}',
         when: enable_rabbitmq | bool }
 
+- name: Apply role rabbitmq (outward)
+  gather_facts: false
+  hosts: outward-rabbitmq
+  roles:
+    - { role: rabbitmq,
+        tags: rabbitmq,
+        project_name: outward_rabbitmq,
+        role_rabbitmq_cluster_cookie: '{{ outward_rabbitmq_cluster_cookie }}',
+        role_rabbitmq_cluster_port: '{{ outward_rabbitmq_cluster_port }}',
+        role_rabbitmq_epmd_port: '{{ outward_rabbitmq_epmd_port }}',
+        role_rabbitmq_groups: outward-rabbitmq,
+        role_rabbitmq_management_port: '{{ outward_rabbitmq_management_port }}',
+        role_rabbitmq_password: '{{ outward_rabbitmq_password }}',
+        role_rabbitmq_port: '{{ outward_rabbitmq_port }}',
+        role_rabbitmq_user: '{{ outward_rabbitmq_user }}',
+        when: enable_outward_rabbitmq | bool }
+
 - name: Apply role etcd
   gather_facts: false
   hosts: etcd

etc/kolla/passwords.yml

@@ -180,6 +180,8 @@ gnocchi_user_id:
 ####################
 rabbitmq_password:
 rabbitmq_cluster_cookie:
+outward_rabbitmq_password:
+outward_rabbitmq_cluster_cookie:
 
 ####################
 # HAProxy options

releasenotes/notes/outward-rabbitmq-0024957af87a7c7e.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Add a new 'outward facing' rabbitmq, for services
+    which require a user facing message queue such as
+    Murano or Trove.