Merge "Fix rabbitmq playbook to allow upgrades"

playbooks/rabbitmq-install.yml

@@ -15,10 +15,13 @@
 
 - name: Install rabbitmq server
   hosts: rabbitmq_all
-  max_fail_percentage: 20
+  max_fail_percentage: 0
   user: root
   roles:
-    - { role: "rabbitmq_server", tags: [ "rabbitmq-server" ] }
+    - role: "rabbitmq_server"
+      tags:
+        - "rabbitmq-server"
+        - "upgrade-rabbitmq-server"
     - role: "rsyslog_client"
       rsyslog_client_log_dir: "/var/log/rabbitmq"
       rsyslog_client_config_name: "99-rabbitmq-rsyslog-client.conf"

playbooks/roles/rabbitmq_server/defaults/main.yml

@@ -18,7 +18,16 @@ is_metal: true
 
 rabbitmq_primary_cluster_node: "{{ groups['rabbitmq_all'][0] }}"
 
-rabbitmq_package_url: https://www.rabbitmq.com/releases/rabbitmq-server/v3.5.3/rabbitmq-server_3.5.3-1_all.deb
+# Upgrading the RabbitMQ package requires shutting down the cluster. This variable makes upgrading
+#  the version an explicit action.
+rabbitmq_upgrade: false
+
+# If the user does not want to upgrade but needs to rerun the playbooks for any reason the
+#  upgrade/version state can be ignored by setting `rabbitmq_ignore_version_state=true`
+rabbitmq_ignore_version_state: false
+
+rabbitmq_package_url: "https://www.rabbitmq.com/releases/rabbitmq-server/v3.5.3/rabbitmq-server_3.5.3-1_all.deb"
+rabbitmq_package_version: "{{ rabbitmq_package_url.split('/')[-1].split('_')[1] }}"
 rabbitmq_package_sha256: "0abcae49842c21242da66f1bc02fe4bf73dc3c3c38419861e08bf1fa4ac74cb1"
 rabbitmq_package_path: "/opt/rabbitmq-server.deb"
 

playbooks/roles/rabbitmq_server/files/policy-rc.d

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "All runlevel operations denied by policy" >&2
+exit 101

playbooks/roles/rabbitmq_server/tasks/main.yml

@@ -15,17 +15,14 @@
 
 
 - include: rabbitmq_pre_install.yml
+
+# The install process will be skipped if `rabbitmq_ignore_version_state=true`
 - include: rabbitmq_install.yml
+  when: not rabbitmq_ignore_version_state | bool
+
 - include: rabbitmq_set_cookie.yml
 - include: rabbitmq_post_install.yml
-
-- include: rabbitmq_cluster_create.yml
-  when: >
-    inventory_hostname == groups['rabbitmq_all'][0]
-
-- include: rabbitmq_cluster_join.yml
-  when: >
-    inventory_hostname != groups['rabbitmq_all'][0]
+- include: rabbitmq_cluster.yml
 
 - name: Flush handlers
   meta: flush_handlers

playbooks/roles/rabbitmq_server/tasks/rabbitmq_cluster.yml

@@ -0,0 +1,22 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- include: rabbitmq_cluster_create.yml
+  when: >
+    inventory_hostname == rabbitmq_primary_cluster_node
+
+- include: rabbitmq_cluster_join.yml
+  when: >
+    inventory_hostname != rabbitmq_primary_cluster_node

playbooks/roles/rabbitmq_server/tasks/rabbitmq_install.yml

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- include: rabbitmq_upgrade_check.yml
+
 - name: Update apt sources
   apt:
     update_cache: yes
@@ -54,3 +56,13 @@
   with_items: rabbitmq_pip_packages
   tags:
     - rabbitmq-pip-packages
+
+- name: Ensure there are no policy-rc files
+  file:
+    path: "/usr/sbin/policy-rc.d"
+    state: absent
+  tags:
+    - rabbitmq-upgrade
+    - rabbitmq-policy-file
+
+- include: rabbitmq_started.yml

playbooks/roles/rabbitmq_server/tasks/rabbitmq_pre_install.yml

@@ -13,6 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Remove package rabbit if found for upgrade
+  file:
+    path: "{{ rabbitmq_package_path }}"
+    state: absent
+  tags:
+    - rabbitmq-upgrade
+    - rabbitmq-package-deb
+    - rabbitmq-package-deb-get
+
 - name: Download the RabbitMQ package
   get_url:
     url: "{{ rabbitmq_package_url }}"

playbooks/roles/rabbitmq_server/tasks/rabbitmq_restart.yml

@@ -13,10 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Restart rabbitmq-server
-  service:
-    name: "rabbitmq-server"
-    enabled: "yes"
-    state: "restarted"
-  tags:
-    - rabbitmq-restart
+- include: rabbitmq_stopped.yml
+- include: rabbitmq_started.yml

playbooks/roles/rabbitmq_server/tasks/rabbitmq_set_cookie.yml

@@ -21,14 +21,10 @@
   tags:
     - rabbitmq-config
 
-- name: Stop rabbitmq-server
-  service:
-    name: rabbitmq-server
-    state: stopped
+- include: rabbitmq_stopped.yml
   when: tmp_rabbit_cookie.stdout != rabbitmq_cookie_token
   tags:
     - rabbitmq-config
-    - rabbitmq-stop
 
 - name: Set rabbit cookie
   copy:
@@ -42,5 +38,7 @@
   tags:
     - rabbitmq-config
 
-- include: rabbitmq_restart.yml
+- include: rabbitmq_started.yml
   when: tmp_rabbit_cookie.stdout != rabbitmq_cookie_token
+  tags:
+    - rabbitmq-config

playbooks/roles/rabbitmq_server/tasks/rabbitmq_started.yml

@@ -0,0 +1,38 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "Ensure RabbitMQ node [0] is started"
+  service:
+    name: rabbitmq-server
+    state: started
+  failed_when: false
+  when: >
+    inventory_hostname == rabbitmq_primary_cluster_node
+  tags:
+    - rabbitmq-server-started
+    - rabbitmq-upgrade
+    - rabbitmq-start
+
+- name: "Ensure RabbitMQ nodes [1:] are started"
+  service:
+    name: rabbitmq-server
+    state: started
+  failed_when: false
+  when: >
+    inventory_hostname != rabbitmq_primary_cluster_node
+  tags:
+    - rabbitmq-server-started
+    - rabbitmq-upgrade
+    - rabbitmq-start

playbooks/roles/rabbitmq_server/tasks/rabbitmq_stopped.yml

@@ -0,0 +1,38 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "Ensure RabbitMQ nodes [1:] are stopped"
+  service:
+    name: rabbitmq-server
+    state: stopped
+  failed_when: false
+  when: >
+    inventory_hostname != rabbitmq_primary_cluster_node
+  tags:
+    - rabbitmq-server-stopped
+    - rabbitmq-upgrade
+    - rabbitmq-stop
+
+- name: "Ensure RabbitMQ node [0] is stopped"
+  service:
+    name: rabbitmq-server
+    state: stopped
+  failed_when: false
+  when: >
+    inventory_hostname == rabbitmq_primary_cluster_node
+  tags:
+    - rabbitmq-server-stopped
+    - rabbitmq-upgrade
+    - rabbitmq-stop

playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_check.yml

@@ -0,0 +1,39 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Get version of installed RabbitMQ package
+  shell: |
+    dpkg -l | grep rabbitmq-server
+  failed_when: false
+  register: installed_rabbitmq
+  when:
+    - not rabbitmq_upgrade | bool
+  tags:
+    - rabbitmq-package-deb
+    - rabbitmq-apt-packages
+
+- name: Compare installed version of RabbitMQ with new version variable
+  fail:
+    msg: "To install a new major/minor version of RabbitMQ set '-e rabbitmq_upgrade=true'."
+  when: >
+    not rabbitmq_upgrade | bool and
+    installed_rabbitmq.rc == 0 and
+    installed_rabbitmq.stdout.split()[2] != rabbitmq_package_version
+  tags:
+    - rabbitmq-package-deb
+    - rabbitmq-apt-packages
+
+- include: rabbitmq_upgrade_prep.yml
+  when: rabbitmq_upgrade | bool

playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_prep.yml

@@ -0,0 +1,33 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Prevent RabbitMQ from starting on install
+  copy:
+    src: "policy-rc.d"
+    dest: "/usr/sbin/policy-rc.d"
+    mode: "0755"
+    backup: yes
+  tags:
+    - rabbitmq-policy-file
+    - rabbitmq-upgrade
+
+- include: rabbitmq_stopped.yml
+
+- name: Ensure erlang epmd is stopped
+  shell: |
+    pkill $(pgrep -lf "erlang" | awk '{print $2}')
+  ignore_errors: true
+  tags:
+    - rabbitmq-upgrade

scripts/run-upgrade.sh

@@ -625,7 +625,7 @@ pushd playbooks
   RUN_TASKS+=("openstack-hosts-setup.yml")
 
   # Now run the infrastructure setup
-  RUN_TASKS+=("setup-infrastructure.yml")
+  RUN_TASKS+=("-e 'rabbitmq_upgrade=true' setup-infrastructure.yml")
 
   # Now upgrade the rest of OpenStack
   RUN_TASKS+=("setup-openstack.yml")