From 3f57d7380d8ffc6908750efb02ff2d0221839786 Mon Sep 17 00:00:00 2001 From: Jiri Stransky Date: Thu, 29 Nov 2018 12:02:21 +0100 Subject: [PATCH] Fix pacemaker tagging race condition Change I81bc48b53068c3a5ed90266a4fd3e62bfb017835 moved image fetching and tagging for pacemaker-managed services from step 1 to step 2. This is also a step when the services are started, which probably introduced a race condition for environments where pacemaker cluster consists of more than one machine. During the deployment you can get a lot of pcmk failures like: failed to pull image 192.168.24.1:8787/tripleomaster/centos-binary-mariadb:pcmklatest This only happens on non-bootstrap nodes. On bootstrap node the order is still correct, first download and tag image, and then start the pcmk resources. However, if non-bootstrap nodes are slower with downloading and tagging, pacemaker there might start the resources before the images are tagged (as the starting of resources is controlled globally from bootstrap node). Change-Id: Id669cc9a296a8366c7c80a5ee509bdb964b62a04 Closes-Bug: #1805826 --- docker/services/pacemaker/cinder-backup.yaml | 2 +- docker/services/pacemaker/cinder-volume.yaml | 2 +- docker/services/pacemaker/database/mysql.yaml | 2 +- docker/services/pacemaker/database/redis.yaml | 2 +- docker/services/pacemaker/haproxy.yaml | 2 +- docker/services/pacemaker/manila-share.yaml | 2 +- docker/services/pacemaker/notify-rabbitmq.yaml | 2 +- docker/services/pacemaker/ovn-dbs.yaml | 2 +- docker/services/pacemaker/rabbitmq.yaml | 2 +- docker/services/pacemaker/rpc-rabbitmq.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/services/pacemaker/cinder-backup.yaml b/docker/services/pacemaker/cinder-backup.yaml index 3617ac708d..8e3dfd9698 100644 --- a/docker/services/pacemaker/cinder-backup.yaml +++ b/docker/services/pacemaker/cinder-backup.yaml @@ -247,7 +247,7 @@ outputs: state: directory deploy_steps_tasks: - name: Cinder Backup tag container image for pacemaker - when: step|int == 2 + when: step|int == 1 import_role: name: tripleo-container-tag vars: diff --git a/docker/services/pacemaker/cinder-volume.yaml b/docker/services/pacemaker/cinder-volume.yaml index e94fe42bf2..7edc3b8732 100644 --- a/docker/services/pacemaker/cinder-volume.yaml +++ b/docker/services/pacemaker/cinder-volume.yaml @@ -213,7 +213,7 @@ outputs: host_prep_tasks: {get_attr: [CinderCommon, cinder_volume_host_prep_tasks]} deploy_steps_tasks: - name: Cinder Volume tag container image for pacemaker - when: step|int == 2 + when: step|int == 1 import_role: name: tripleo-container-tag vars: diff --git a/docker/services/pacemaker/database/mysql.yaml b/docker/services/pacemaker/database/mysql.yaml index a07df88222..a0f744d33d 100644 --- a/docker/services/pacemaker/database/mysql.yaml +++ b/docker/services/pacemaker/database/mysql.yaml @@ -307,7 +307,7 @@ outputs: get_attr: [MysqlPuppetBase, role_data, metadata_settings] deploy_steps_tasks: - name: MySQL tag container image for pacemaker - when: step|int == 2 + when: step|int == 1 import_role: name: tripleo-container-tag vars: diff --git a/docker/services/pacemaker/database/redis.yaml b/docker/services/pacemaker/database/redis.yaml index 68a71fba4b..f24b41809f 100644 --- a/docker/services/pacemaker/database/redis.yaml +++ b/docker/services/pacemaker/database/redis.yaml @@ -286,7 +286,7 @@ outputs: ignore_errors: true deploy_steps_tasks: - name: Redis tag container image for pacemaker - when: step|int == 2 + when: step|int == 1 import_role: name: tripleo-container-tag vars: diff --git a/docker/services/pacemaker/haproxy.yaml b/docker/services/pacemaker/haproxy.yaml index a96617c933..c143d3b7e6 100644 --- a/docker/services/pacemaker/haproxy.yaml +++ b/docker/services/pacemaker/haproxy.yaml @@ -290,7 +290,7 @@ outputs: get_attr: [HAProxyBase, role_data, metadata_settings] deploy_steps_tasks: - name: HAproxy tag container image for pacemaker - when: step|int == 2 + when: step|int == 1 import_role: name: tripleo-container-tag vars: diff --git a/docker/services/pacemaker/manila-share.yaml b/docker/services/pacemaker/manila-share.yaml index 73b28a8fe9..b4395201cf 100644 --- a/docker/services/pacemaker/manila-share.yaml +++ b/docker/services/pacemaker/manila-share.yaml @@ -214,7 +214,7 @@ outputs: state: directory deploy_steps_tasks: - name: Manila Share tag container image for pacemaker - when: step|int == 2 + when: step|int == 1 import_role: name: tripleo-container-tag vars: diff --git a/docker/services/pacemaker/notify-rabbitmq.yaml b/docker/services/pacemaker/notify-rabbitmq.yaml index e57e852213..74efe44963 100644 --- a/docker/services/pacemaker/notify-rabbitmq.yaml +++ b/docker/services/pacemaker/notify-rabbitmq.yaml @@ -260,7 +260,7 @@ outputs: get_attr: [RabbitmqBase, role_data, metadata_settings] deploy_steps_tasks: - name: RabbitMQ tag container image for pacemaker - when: step|int == 2 + when: step|int == 1 import_role: name: tripleo-container-tag vars: diff --git a/docker/services/pacemaker/ovn-dbs.yaml b/docker/services/pacemaker/ovn-dbs.yaml index 1b2fb63ad6..cfdd8e74d5 100644 --- a/docker/services/pacemaker/ovn-dbs.yaml +++ b/docker/services/pacemaker/ovn-dbs.yaml @@ -196,7 +196,7 @@ outputs: ignore_errors: true deploy_steps_tasks: - name: OVN DBS tag container image for pacemaker - when: step|int == 2 + when: step|int == 1 import_role: name: tripleo-container-tag vars: diff --git a/docker/services/pacemaker/rabbitmq.yaml b/docker/services/pacemaker/rabbitmq.yaml index a4b3dc66b0..28e2ac11e7 100644 --- a/docker/services/pacemaker/rabbitmq.yaml +++ b/docker/services/pacemaker/rabbitmq.yaml @@ -259,7 +259,7 @@ outputs: get_attr: [RabbitmqBase, role_data, metadata_settings] deploy_steps_tasks: - name: RabbitMQ tag container image for pacemaker - when: step|int == 2 + when: step|int == 1 import_role: name: tripleo-container-tag vars: diff --git a/docker/services/pacemaker/rpc-rabbitmq.yaml b/docker/services/pacemaker/rpc-rabbitmq.yaml index 6b137ac1bc..d3e9d61e96 100644 --- a/docker/services/pacemaker/rpc-rabbitmq.yaml +++ b/docker/services/pacemaker/rpc-rabbitmq.yaml @@ -260,7 +260,7 @@ outputs: for pid in $(pgrep epmd --ns 1 --nslist pid); do kill $pid; done deploy_steps_tasks: - name: RabbitMQ tag container image for pacemaker - when: step|int == 2 + when: step|int == 1 import_role: name: tripleo-container-tag vars: