From 6ea86e6274bedaff955d734222c9f4b1cc2c3a2a Mon Sep 17 00:00:00 2001 From: git-harry Date: Thu, 16 Jul 2015 11:01:13 +0100 Subject: [PATCH] Fix rabbitmq playbook to allow upgrades The rabbitmq playbook is designed to run in parallel across the cluster. This causes an issue when upgrading rabbitmq to a new major or minor version because RabbitMQ does not support doing an online migration of datasets between major versions. while a minor release can be upgrade while online it is recommended to bring down the cluster to do any upgrade actions. The current configuration takes no account of this. Reference: https://www.rabbitmq.com/clustering.html#upgrading for further details. * A new variable has been added called `rabbitmq_upgrade`. This is set to false by default to prevent a new version being installed unintentionally. To run the upgrade, which will shutdown the cluster, the variable can be set to true on the commandline: Example: openstack-ansible -e rabbitmq_upgrade=true \ rabbitmq-install.yml * A new variable has been added called `rabbitmq_ignore_version_state` which can be set "true" to ignore the package and version state tasks. This has been provided to allow a deployer to rerun the plays in an environment where the playbooks have been upgraded and the default version of rabbitmq has changed within the role and the deployer has elected to upgraded the installation at that time. This will ensure a deployer is able to recluster an environment as needed without effecting the package state. Example: openstack-ansible -e rabbitmq_ignore_version_state=true \ rabbitmq-install.yml * A new variable has been added `rabbitmq_primary_cluster_node` which allows a deployer to elect / set the primary cluster node in an environment. This variable is used to determine the restart order of RabbitMQ nodes. IE this will be the last node down and first one up in an environment. By default this variable is set to: rabbitmq_primary_cluster_node: "{{ groups['rabbitmq_all'][0] }}" scripts/run-upgrade.sh has been modified to pass 'rabbitmq_upgrade=true' on the command line so that RabbitMQ can be upgraded as part of the upgrade between OpenStack versions. DocImpact Change-Id: I17d4429b9b94d47c1578dd58a2fb20698d1fe02e Closes-bug: #1474992 --- playbooks/rabbitmq-install.yml | 7 +++- .../roles/rabbitmq_server/defaults/main.yml | 11 +++++- .../roles/rabbitmq_server/files/policy-rc.d | 3 ++ .../roles/rabbitmq_server/tasks/main.yml | 13 +++---- .../tasks/rabbitmq_cluster.yml | 22 +++++++++++ .../tasks/rabbitmq_install.yml | 12 ++++++ .../tasks/rabbitmq_pre_install.yml | 9 +++++ .../tasks/rabbitmq_restart.yml | 9 +---- .../tasks/rabbitmq_set_cookie.yml | 10 ++--- .../tasks/rabbitmq_started.yml | 38 ++++++++++++++++++ .../tasks/rabbitmq_stopped.yml | 38 ++++++++++++++++++ .../tasks/rabbitmq_upgrade_check.yml | 39 +++++++++++++++++++ .../tasks/rabbitmq_upgrade_prep.yml | 33 ++++++++++++++++ scripts/run-upgrade.sh | 2 +- 14 files changed, 221 insertions(+), 25 deletions(-) create mode 100644 playbooks/roles/rabbitmq_server/files/policy-rc.d create mode 100644 playbooks/roles/rabbitmq_server/tasks/rabbitmq_cluster.yml create mode 100644 playbooks/roles/rabbitmq_server/tasks/rabbitmq_started.yml create mode 100644 playbooks/roles/rabbitmq_server/tasks/rabbitmq_stopped.yml create mode 100644 playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_check.yml create mode 100644 playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_prep.yml diff --git a/playbooks/rabbitmq-install.yml b/playbooks/rabbitmq-install.yml index 3b2ca1ff00..c8370e5767 100644 --- a/playbooks/rabbitmq-install.yml +++ b/playbooks/rabbitmq-install.yml @@ -15,10 +15,13 @@ - name: Install rabbitmq server hosts: rabbitmq_all - max_fail_percentage: 20 + max_fail_percentage: 0 user: root roles: - - { role: "rabbitmq_server", tags: [ "rabbitmq-server" ] } + - role: "rabbitmq_server" + tags: + - "rabbitmq-server" + - "upgrade-rabbitmq-server" - role: "rsyslog_client" rsyslog_client_log_dir: "/var/log/rabbitmq" rsyslog_client_config_name: "99-rabbitmq-rsyslog-client.conf" diff --git a/playbooks/roles/rabbitmq_server/defaults/main.yml b/playbooks/roles/rabbitmq_server/defaults/main.yml index be7747f419..72cf65efd2 100644 --- a/playbooks/roles/rabbitmq_server/defaults/main.yml +++ b/playbooks/roles/rabbitmq_server/defaults/main.yml @@ -18,7 +18,16 @@ is_metal: true rabbitmq_primary_cluster_node: "{{ groups['rabbitmq_all'][0] }}" -rabbitmq_package_url: https://www.rabbitmq.com/releases/rabbitmq-server/v3.5.3/rabbitmq-server_3.5.3-1_all.deb +# Upgrading the RabbitMQ package requires shutting down the cluster. This variable makes upgrading +# the version an explicit action. +rabbitmq_upgrade: false + +# If the user does not want to upgrade but needs to rerun the playbooks for any reason the +# upgrade/version state can be ignored by setting `rabbitmq_ignore_version_state=true` +rabbitmq_ignore_version_state: false + +rabbitmq_package_url: "https://www.rabbitmq.com/releases/rabbitmq-server/v3.5.3/rabbitmq-server_3.5.3-1_all.deb" +rabbitmq_package_version: "{{ rabbitmq_package_url.split('/')[-1].split('_')[1] }}" rabbitmq_package_sha256: "0abcae49842c21242da66f1bc02fe4bf73dc3c3c38419861e08bf1fa4ac74cb1" rabbitmq_package_path: "/opt/rabbitmq-server.deb" diff --git a/playbooks/roles/rabbitmq_server/files/policy-rc.d b/playbooks/roles/rabbitmq_server/files/policy-rc.d new file mode 100644 index 0000000000..ec21cabfa6 --- /dev/null +++ b/playbooks/roles/rabbitmq_server/files/policy-rc.d @@ -0,0 +1,3 @@ +#!/bin/sh +echo "All runlevel operations denied by policy" >&2 +exit 101 diff --git a/playbooks/roles/rabbitmq_server/tasks/main.yml b/playbooks/roles/rabbitmq_server/tasks/main.yml index 2a5695dbb2..5f5c4788d9 100644 --- a/playbooks/roles/rabbitmq_server/tasks/main.yml +++ b/playbooks/roles/rabbitmq_server/tasks/main.yml @@ -15,17 +15,14 @@ - include: rabbitmq_pre_install.yml + +# The install process will be skipped if `rabbitmq_ignore_version_state=true` - include: rabbitmq_install.yml + when: not rabbitmq_ignore_version_state | bool + - include: rabbitmq_set_cookie.yml - include: rabbitmq_post_install.yml - -- include: rabbitmq_cluster_create.yml - when: > - inventory_hostname == groups['rabbitmq_all'][0] - -- include: rabbitmq_cluster_join.yml - when: > - inventory_hostname != groups['rabbitmq_all'][0] +- include: rabbitmq_cluster.yml - name: Flush handlers meta: flush_handlers diff --git a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_cluster.yml b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_cluster.yml new file mode 100644 index 0000000000..bac4246053 --- /dev/null +++ b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_cluster.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: rabbitmq_cluster_create.yml + when: > + inventory_hostname == rabbitmq_primary_cluster_node + +- include: rabbitmq_cluster_join.yml + when: > + inventory_hostname != rabbitmq_primary_cluster_node diff --git a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_install.yml b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_install.yml index 63e51cdc60..e39d83f266 100644 --- a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_install.yml +++ b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_install.yml @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +- include: rabbitmq_upgrade_check.yml + - name: Update apt sources apt: update_cache: yes @@ -54,3 +56,13 @@ with_items: rabbitmq_pip_packages tags: - rabbitmq-pip-packages + +- name: Ensure there are no policy-rc files + file: + path: "/usr/sbin/policy-rc.d" + state: absent + tags: + - rabbitmq-upgrade + - rabbitmq-policy-file + +- include: rabbitmq_started.yml diff --git a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_pre_install.yml b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_pre_install.yml index f9019b9586..d3468a41b7 100644 --- a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_pre_install.yml +++ b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_pre_install.yml @@ -13,6 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Remove package rabbit if found for upgrade + file: + path: "{{ rabbitmq_package_path }}" + state: absent + tags: + - rabbitmq-upgrade + - rabbitmq-package-deb + - rabbitmq-package-deb-get + - name: Download the RabbitMQ package get_url: url: "{{ rabbitmq_package_url }}" diff --git a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_restart.yml b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_restart.yml index 595c3f71ba..aec2cea3fd 100644 --- a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_restart.yml +++ b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_restart.yml @@ -13,10 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Restart rabbitmq-server - service: - name: "rabbitmq-server" - enabled: "yes" - state: "restarted" - tags: - - rabbitmq-restart +- include: rabbitmq_stopped.yml +- include: rabbitmq_started.yml diff --git a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_set_cookie.yml b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_set_cookie.yml index 6dcff38ace..49c56a0629 100644 --- a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_set_cookie.yml +++ b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_set_cookie.yml @@ -21,14 +21,10 @@ tags: - rabbitmq-config -- name: Stop rabbitmq-server - service: - name: rabbitmq-server - state: stopped +- include: rabbitmq_stopped.yml when: tmp_rabbit_cookie.stdout != rabbitmq_cookie_token tags: - rabbitmq-config - - rabbitmq-stop - name: Set rabbit cookie copy: @@ -42,5 +38,7 @@ tags: - rabbitmq-config -- include: rabbitmq_restart.yml +- include: rabbitmq_started.yml when: tmp_rabbit_cookie.stdout != rabbitmq_cookie_token + tags: + - rabbitmq-config diff --git a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_started.yml b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_started.yml new file mode 100644 index 0000000000..0bbef5dc53 --- /dev/null +++ b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_started.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "Ensure RabbitMQ node [0] is started" + service: + name: rabbitmq-server + state: started + failed_when: false + when: > + inventory_hostname == rabbitmq_primary_cluster_node + tags: + - rabbitmq-server-started + - rabbitmq-upgrade + - rabbitmq-start + +- name: "Ensure RabbitMQ nodes [1:] are started" + service: + name: rabbitmq-server + state: started + failed_when: false + when: > + inventory_hostname != rabbitmq_primary_cluster_node + tags: + - rabbitmq-server-started + - rabbitmq-upgrade + - rabbitmq-start diff --git a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_stopped.yml b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_stopped.yml new file mode 100644 index 0000000000..404c9ead2b --- /dev/null +++ b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_stopped.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "Ensure RabbitMQ nodes [1:] are stopped" + service: + name: rabbitmq-server + state: stopped + failed_when: false + when: > + inventory_hostname != rabbitmq_primary_cluster_node + tags: + - rabbitmq-server-stopped + - rabbitmq-upgrade + - rabbitmq-stop + +- name: "Ensure RabbitMQ node [0] is stopped" + service: + name: rabbitmq-server + state: stopped + failed_when: false + when: > + inventory_hostname == rabbitmq_primary_cluster_node + tags: + - rabbitmq-server-stopped + - rabbitmq-upgrade + - rabbitmq-stop diff --git a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_check.yml b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_check.yml new file mode 100644 index 0000000000..8513adfae6 --- /dev/null +++ b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_check.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Get version of installed RabbitMQ package + shell: | + dpkg -l | grep rabbitmq-server + failed_when: false + register: installed_rabbitmq + when: + - not rabbitmq_upgrade | bool + tags: + - rabbitmq-package-deb + - rabbitmq-apt-packages + +- name: Compare installed version of RabbitMQ with new version variable + fail: + msg: "To install a new major/minor version of RabbitMQ set '-e rabbitmq_upgrade=true'." + when: > + not rabbitmq_upgrade | bool and + installed_rabbitmq.rc == 0 and + installed_rabbitmq.stdout.split()[2] != rabbitmq_package_version + tags: + - rabbitmq-package-deb + - rabbitmq-apt-packages + +- include: rabbitmq_upgrade_prep.yml + when: rabbitmq_upgrade | bool diff --git a/playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_prep.yml b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_prep.yml new file mode 100644 index 0000000000..4c9336c7a8 --- /dev/null +++ b/playbooks/roles/rabbitmq_server/tasks/rabbitmq_upgrade_prep.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prevent RabbitMQ from starting on install + copy: + src: "policy-rc.d" + dest: "/usr/sbin/policy-rc.d" + mode: "0755" + backup: yes + tags: + - rabbitmq-policy-file + - rabbitmq-upgrade + +- include: rabbitmq_stopped.yml + +- name: Ensure erlang epmd is stopped + shell: | + pkill $(pgrep -lf "erlang" | awk '{print $2}') + ignore_errors: true + tags: + - rabbitmq-upgrade diff --git a/scripts/run-upgrade.sh b/scripts/run-upgrade.sh index 113671e14d..1ebd019d8a 100755 --- a/scripts/run-upgrade.sh +++ b/scripts/run-upgrade.sh @@ -625,7 +625,7 @@ pushd playbooks RUN_TASKS+=("openstack-hosts-setup.yml") # Now run the infrastructure setup - RUN_TASKS+=("setup-infrastructure.yml") + RUN_TASKS+=("-e 'rabbitmq_upgrade=true' setup-infrastructure.yml") # Now upgrade the rest of OpenStack RUN_TASKS+=("setup-openstack.yml")