Refactor rabbitmq ordering

- Refactor Service_status['rabbitmq'] in pacemaker and non pacemaker
  mode. As a part cmd_check was changed to 'rabbitmqctl
  node_health_check && rabbitmqctl cluster_status' as cluster_status may
  fail due to issues on other nodes. Also it doesn't have timeout thus
  operation may hang up. Instead node_health_check will check the node
  and check cluster on success.
- Staging::File as MODULES-1650 has been merged
- Specify ordering between rabbitmq_user_permissions and rabbitmq_user
- Write test that verifies that service is before the actual user

Change-Id: I876ab09d687916195dcb82633f0242fc823c0357
Closes-Bug: #1652268
Signed-off-by: Sergii Golovatiuk <sgolovatiuk@mirantis.com>

deployment/puppet/cluster/manifests/rabbitmq_ocf.pp

@@ -152,10 +152,10 @@ class cluster::rabbitmq_ocf (
   }
 
   $complex_metadata     = {
-    'notify'      => 'true',
+    'notify'      => true,
     # We shouldn't enable ordered start for parallel start of RA.
-    'ordered'     => 'false',
-    'interleave'  => 'true',
+    'ordered'     => false,
+    'interleave'  => true,
     'master-max'  => '1',
     'master-node-max' => '1',
     'target-role' => 'Master',
@@ -195,16 +195,23 @@ class cluster::rabbitmq_ocf (
   }
 
   pacemaker::service { $service_name :
-    primitive_type      => $primitive_type,
-    complex_type        => 'master',
-    complex_metadata    => $complex_metadata,
-    metadata            => $metadata,
-    operations          => $operations,
-    parameters          => $parameters,
-    #    ocf_script_file     => $ocf_script_file,
+    primitive_type   => $primitive_type,
+    complex_type     => 'master',
+    complex_metadata => $complex_metadata,
+    metadata         => $metadata,
+    operations       => $operations,
+    parameters       => $parameters,
+    #ocf_script_file => $ocf_script_file,
   }
 
-  ensure_resource('service_status', ['rabbitmq'], { 'ensure' => 'online', check_cmd => 'rabbitmqctl cluster_status'})
+  if !defined(Service_status['rabbitmq']) {
+    ensure_resource('service_status', ['rabbitmq'],
+      { 'ensure' => 'online', 'check_cmd' => 'rabbitmqctl node_health_check && rabbitmqctl cluster_status'})
+  } else {
+    Service_status<| title == 'rabbitmq' |> {
+        check_cmd => 'rabbitmqctl node_health_check && rabbitmqctl cluster_status',
+    }
+  }
 
-  Service[$service_name] -> Service_status['rabbitmq'] -> Rabbitmq_user <||>
+  Service[$service_name] -> Service_status['rabbitmq']
 }

deployment/puppet/osnailyfacter/manifests/rabbitmq/rabbitmq.pp

@@ -150,18 +150,6 @@ class osnailyfacter::rabbitmq::rabbitmq {
         tcp_keepalive               => true,
       }
 
-      # NOTE(bogdando) retries for the rabbitmqadmin curl command, unmerged MODULES-1650
-      Staging::File <| title == 'rabbitmqadmin' |> {
-          tries       => 30,
-          try_sleep   => 6,
-      }
-
-      if $rabbitmq_admin_enabled {
-        ensure_resource('service_status', ['rabbitmq'], { 'ensure' => 'online', check_cmd => 'rabbitmqctl cluster_status'})
-
-        Service_status['rabbitmq'] -> Staging::File['rabbitmqadmin']
-      }
-
       # Make sure the various providers have their requirements in place.
       Class['::rabbitmq::install'] -> File['/etc/rabbitmq'] -> Rabbitmq_plugin<| |> -> Rabbitmq_exchange<| |>
 
@@ -169,7 +157,7 @@ class osnailyfacter::rabbitmq::rabbitmq {
         admin    => true,
         password => $rabbit_hash['password'],
         provider => 'rabbitmqctl',
-      }
+      } ->
 
       rabbitmq_user_permissions { "${rabbit_hash['user']}@/":
         configure_permission => '.*',
@@ -183,6 +171,7 @@ class osnailyfacter::rabbitmq::rabbitmq {
         provider => 'rabbitmqctl',
       }
 
+
       if ($use_pacemaker) {
         # Install rabbit-fence daemon
         class { '::cluster::rabbitmq_fence':
@@ -216,6 +205,17 @@ class osnailyfacter::rabbitmq::rabbitmq {
         }
       }
 
+      if !defined(Service_status['rabbitmq']) {
+        ensure_resource('service_status', ['rabbitmq'],
+                        { 'ensure' => 'online', 'check_cmd' => 'rabbitmqctl node_health_check'})
+      }
+
+      Service_status['rabbitmq'] -> Rabbitmq_user <||>
+
+      if $rabbitmq_admin_enabled {
+        Service_status['rabbitmq'] -> Staging::File['rabbitmqadmin']
+      }
+
       include ::rabbitmq::params
       tweaks::ubuntu_service_override { 'rabbitmq-server':
         package_name => $rabbitmq::params::package_name,

tests/noop/spec/hosts/rabbitmq/rabbitmq_spec.rb

@@ -135,6 +135,10 @@ describe manifest do
       )
     end
 
+    it 'should configure rabbitmq service before rabbitmq user is setup' do
+      is_expected.to contain_service_status('rabbitmq').that_comes_before("Rabbitmq_user[#{params[:default_user]}]")
+    end
+
     if rabbitmq_admin_enabled
       it 'should check rabbitmq status' do
         should contain_service_status('rabbitmq').that_comes_before('Staging::File[rabbitmqadmin]')