Retry if the pacemaker_resource commands failed

Add a retry when the pacemaker_resource command
wasn't apply correctly, more info here:
https://bugzilla.redhat.com/show_bug.cgi?id=1482116

This is the same approach puppet-pacemaker uses
and provides eventual consistency when multiple
nodes change the cluster CIB concurrently.

This change depends-on :
https://review.gerrithub.io/375982

The return code is not available in the current
ansible-pacemaker package.

Change-Id: I8da03f5c4a6d442617b81be5793a9724cc8842bf

docker/services/pacemaker/cinder-backup.yaml

@@ -188,6 +188,9 @@ outputs:
             resource: openstack-cinder-backup
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped openstack-cinder-backup cluster resource.
           tags: step2
@@ -195,6 +198,9 @@ outputs:
             resource: openstack-cinder-backup
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable cinder_backup service
           tags: step2

docker/services/pacemaker/cinder-volume.yaml

@@ -206,6 +206,9 @@ outputs:
             resource: openstack-cinder-volume
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped openstack-cinder-volume cluster resource.
           tags: step2
@@ -213,6 +216,9 @@ outputs:
             resource: openstack-cinder-volume
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable cinder_volume service from boot
           tags: step2

docker/services/pacemaker/database/mysql.yaml

@@ -272,6 +272,9 @@ outputs:
             resource: galera
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped galera cluster resource.
           tags: step2
@@ -279,6 +282,9 @@ outputs:
             resource: galera
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable mysql service
           tags: step2

docker/services/pacemaker/database/redis.yaml

@@ -187,6 +187,9 @@ outputs:
             resource: {get_attr: [RedisBase, role_data, service_name]}
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped redis cluster resource.
           tags: step2
@@ -194,6 +197,9 @@ outputs:
             resource: {get_attr: [RedisBase, role_data, service_name]}
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable redis service
           tags: step2

docker/services/pacemaker/haproxy.yaml

@@ -230,6 +230,9 @@ outputs:
             resource: {get_attr: [HAProxyBase, role_data, service_name]}
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped haproxy cluster resource.
           tags: step2
@@ -237,4 +240,7 @@ outputs:
             resource: {get_attr: [HAProxyBase, role_data, service_name]}
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node

docker/services/pacemaker/rabbitmq.yaml

@@ -221,6 +221,9 @@ outputs:
             resource: {get_attr: [RabbitmqBase, role_data, service_name]}
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped rabbitmq cluster resource.
           tags: step2
@@ -228,6 +231,9 @@ outputs:
             resource: {get_attr: [RabbitmqBase, role_data, service_name]}
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable rabbitmq service
           tags: step2