From 4d61344c14f348eae3dcc7964c0489e9efe63c1e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 15 Dec 2021 16:07:50 +0000 Subject: [PATCH] cinder: restart services after upgrade This patch is roughly an adaptation of Ia6fc9011ee6f5461f40a1307b72709d769814a79 for cinder. During an upgrade, cinder pins the version of RPC calls to the minimum seen across all services. This ensures that old services do not receive data they cannot handle. After the upgrade is complete, all cinder services are supposed to be reloaded to cause them to check again the RPC versions of services and use the new latest version which should now be supported by all running services. There is a second issue in that it takes some time for the upgraded services to update the cinder services database table with their new version. We need to wait until all cinder services have done this before the restart is performed, otherwise the RPC version cap will remain in place. There is currently no interface in cinder available for checking these versions, so as a workaround we use a configurable delay with a default duration of 30 seconds, as we do for nova. This change restarts all cinder services after an upgrade, after a 30 second delay. Closes-Bug: #1954932 Related-Bug: #1833069 Change-Id: I9164dc589386d2c2d4daf1bf84061b806ba9988d (cherry picked from commit 80a32c3c74cad4d46671faff067b274ffed74bba) --- ansible/roles/cinder/defaults/main.yml | 9 +++++++++ ansible/roles/cinder/handlers/main.yml | 20 +++++++++++++++++++ ansible/roles/cinder/tasks/reload.yml | 10 ++++++++++ ansible/roles/cinder/tasks/upgrade.yml | 2 ++ .../unpin-cinder-rpcs-8eb7e0858a91b9b8.yaml | 6 ++++++ 5 files changed, 47 insertions(+) create mode 100644 ansible/roles/cinder/tasks/reload.yml create mode 100644 releasenotes/notes/unpin-cinder-rpcs-8eb7e0858a91b9b8.yaml diff --git a/ansible/roles/cinder/defaults/main.yml b/ansible/roles/cinder/defaults/main.yml index 9259f241f8..43823153ae 100644 --- a/ansible/roles/cinder/defaults/main.yml +++ b/ansible/roles/cinder/defaults/main.yml @@ -201,6 +201,15 @@ cinder_logging_debug: "{{ openstack_logging_debug }}" openstack_cinder_auth: "{{ openstack_auth }}" +# After upgrading cinder, services will have an RPC version cap in place. We +# need to restart all services in order to allow them to use the latest RPC +# version. Ideally, there would be a way to check whether all cinder services +# are using the latest version, but currently there is not. Instead, wait a +# short time for all cinder services to update the version of their service in +# the database. This seems to take around 10 seconds, but the default is 30 to +# allow room for slowness. +cinder_rpc_version_startup_delay: 30 + #################### # Cinder diff --git a/ansible/roles/cinder/handlers/main.yml b/ansible/roles/cinder/handlers/main.yml index ecd3b8f069..59a08ca8d0 100644 --- a/ansible/roles/cinder/handlers/main.yml +++ b/ansible/roles/cinder/handlers/main.yml @@ -66,3 +66,23 @@ healthcheck: "{{ service.healthcheck | default(omit) }}" when: - kolla_action != "config" + +# NOTE(mgoddard): After upgrading cinder, services will have an RPC version cap +# in place. We need to restart all services in order to allow them to use the +# latest RPC version. Ideally, there would be a way to check whether all cinder +# services are using the latest version, but currently there is not. Instead, +# wait a short time for all cinder services to update the version of their +# service in the database. This seems to take around 10 seconds, but the +# default is 30 to allow room for slowness. + +- name: Wait for cinder services to update service versions + pause: + seconds: "{{ cinder_rpc_version_startup_delay }}" + run_once: true + when: + - kolla_action == 'upgrade' + listen: + - Restart cinder-api container + - Restart cinder-scheduler container + - Restart cinder-volume container + - Restart cinder-backup container diff --git a/ansible/roles/cinder/tasks/reload.yml b/ansible/roles/cinder/tasks/reload.yml new file mode 100644 index 0000000000..a8d6dd3289 --- /dev/null +++ b/ansible/roles/cinder/tasks/reload.yml @@ -0,0 +1,10 @@ +--- +- name: Reload cinder services to remove RPC version pin + vars: + service: "{{ item.value }}" + become: true + kolla_docker: + action: "restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + with_dict: "{{ cinder_services | select_services_enabled_and_mapped_to_host }}" diff --git a/ansible/roles/cinder/tasks/upgrade.yml b/ansible/roles/cinder/tasks/upgrade.yml index fbe1f5a7df..cba28096e2 100644 --- a/ansible/roles/cinder/tasks/upgrade.yml +++ b/ansible/roles/cinder/tasks/upgrade.yml @@ -13,6 +13,8 @@ - name: Flush handlers meta: flush_handlers +- import_tasks: reload.yml + - name: Running Cinder online schema migration vars: cinder_api: "{{ cinder_services['cinder-api'] }}" diff --git a/releasenotes/notes/unpin-cinder-rpcs-8eb7e0858a91b9b8.yaml b/releasenotes/notes/unpin-cinder-rpcs-8eb7e0858a91b9b8.yaml new file mode 100644 index 0000000000..acb8f66bf1 --- /dev/null +++ b/releasenotes/notes/unpin-cinder-rpcs-8eb7e0858a91b9b8.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue with Cinder upgrade where Cinder services would remain + pinned to the previous release's RPC & object versions. `LP#1954932 + `__