From 64b56a53d8f5136c6902ce0fa948317f171a664d Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Tue, 16 Dec 2014 09:53:36 +1100
Subject: [PATCH] Fix rabbitmq retry for error checking

I think this retry check has been broken since we introduced "set -e".
Unfortunately it seems the issue of rabbitmq not starting first-time
persists on centos 7 hosts occasionally, e.g. [1]:

---
 + rabbit_setuser stackrabbit secretrabbit
 + local user=stackrabbit pass=secretrabbit found= out=
 ++ sudo rabbitmqctl list_users
 Error: unable to connect to node 'rabbit@devstack-centos7-rax-iad-100675': nodedown

 DIAGNOSTICS
 ===========

 nodes in question: ['rabbit@devstack-centos7-rax-iad-100675']

 hosts, their running nodes and ports:
 - devstack-centos7-rax-iad-100675: [{rabbitmqctl29293,39511}]

 current node details:
 - node name: 'rabbitmqctl29293@devstack-centos7-rax-iad-100675'
 - home dir: /var/lib/rabbitmq
 - cookie hash: KieJnx1pnllKbHVihGcDqA==
---

Fix up this retry while we investigate [2]

[1] http://logs.openstack.org/64/141864/1/check//check-tempest-dsvm-centos7/4308f0c/logs/devstacklog.txt.gz
[2] https://bugzilla.redhat.com/show_bug.cgi?id=1144100

Change-Id: I11fb3728e08adc1e0f7acca63e5a308d24dce78e
---
 lib/rpc_backend | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/lib/rpc_backend b/lib/rpc_backend
index 6afec370e8..400204abe1 100644
--- a/lib/rpc_backend
+++ b/lib/rpc_backend
@@ -176,17 +176,31 @@ function restart_rpc_backend {
         echo_summary "Starting RabbitMQ"
         # NOTE(bnemec): Retry initial rabbitmq configuration to deal with
         # the fact that sometimes it fails to start properly.
-        # Reference: https://bugzilla.redhat.com/show_bug.cgi?id=1059028
+        # Reference: https://bugzilla.redhat.com/show_bug.cgi?id=1144100
         local i
         for i in `seq 10`; do
+            local rc=0
+
+            [[ $i -eq "10" ]] && die $LINENO "Failed to set rabbitmq password"
+
             if is_fedora || is_suse; then
                 # service is not started by default
                 restart_service rabbitmq-server
             fi
-            rabbit_setuser "$RABBIT_USERID" "$RABBIT_PASSWORD"
+
+            rabbit_setuser "$RABBIT_USERID" "$RABBIT_PASSWORD" || rc=$?
+            if [ $rc -ne 0 ]; then
+                continue
+            fi
+
             # change the rabbit password since the default is "guest"
-            sudo rabbitmqctl change_password $RABBIT_USERID $RABBIT_PASSWORD && break
-            [[ $i -eq "10" ]] && die $LINENO "Failed to set rabbitmq password"
+            sudo rabbitmqctl change_password \
+                $RABBIT_USERID $RABBIT_PASSWORD || rc=$?
+            if [ $rc -ne 0 ]; then
+                continue;
+            fi
+
+            break
         done
         if is_service_enabled n-cell; then
             # Add partitioned access for the child cell