Merge "Redis: Enable master failover"

manifests/database/nosql/redis/sentinel.pp

@@ -24,18 +24,39 @@
 #   Used for firewall purpose.
 #   Default to 26379
 #
+# [*haproxy_monitor_ip*]
+#   (optional) IP on which the HAProxy API is listening on
+#   Used for redis master failover purpose
+#   Default to 127.0.0.1
+#
+# [*haproxy_monitor_port*]
+#   (optional) Port on which the HAProxy API is listening on
+#   Used for redis master failover purpose
+#   Default to 10300
+#
 # [*firewall_settings*]
 #   (optional) Allow to add custom parameters to firewall rules
 #   Should be an hash.
 #   Default to {}
 #
 class cloud::database::nosql::redis::sentinel(
-  $port              = 26379,
+  $port                 = 26379,
-  $firewall_settings = {},
+  $haproxy_monitor_ip   = '127.0.0.1',
+  $haproxy_monitor_port = '10300',
+  $firewall_settings    = {},
 ) {
 
   include ::redis::sentinel
 
+  file { '/bin/redis-notifications.sh':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => template('cloud/database/redis-notifications.sh.erb'),
+    before  => Service['redis-sentinel'],
+  }
+
   if $::cloud::manage_firewall {
     cloud::firewall::rule{ '100 allow redis sentinel access':
       port   => $port,

manifests/database/nosql/redis/server.pp

@@ -19,6 +19,10 @@
 #
 # === Parameters:
 #
+# [*bind_ip*]
+#   (optional) Address on which Redis is listening on
+#   Defaults to '127.0.0.1'
+#
 # [*port*]
 #   (optional) Port where Redis is binded.
 #   Used for firewall purpose.
@@ -30,12 +34,21 @@
 #   Default to {}
 #
 class cloud::database::nosql::redis::server(
+  $bind_ip           = '127.0.0.1',
   $port              = 6379,
   $firewall_settings = {},
 ) {
 
   include ::redis
 
+  @@haproxy::balancermember{"${::fqdn}-redis":
+    listening_service => 'redis_cluster',
+    server_names      => $::hostname,
+    ipaddresses       => $bind_ip,
+    ports             => $port,
+    options           => 'check inter 2000 rise 2 fall 5'
+  }
+
   if $::cloud::manage_firewall {
     cloud::firewall::rule{ '100 allow redis server access':
       port   => $port,

manifests/loadbalancer.pp

@@ -178,6 +178,13 @@
 #   If set to false, no binding will be configure.
 #   Defaults to true
 #
+# [*redis*]
+#   (optional) Enable or not redis binding.
+#   If true, both public and internal will attempt to be created except if vip_internal_ip is set to false.
+#   If set to ['10.0.0.1'], only IP in the array (or in the string) will be configured in the pool. They must be part of keepalived_ip options.
+#   If set to false, no binding will be configure.
+#   Defaults to true
+#
 # [*metadata_api*]
 #   (optional) Enable or not Metadata public binding.
 #   If true, both public and internal will attempt to be created except if vip_internal_ip is set to false.
@@ -352,6 +359,11 @@
 #   service configuration block.
 #   Defaults to []
 #
+# [*redis_bind_options*]
+#   (optional) A hash of options that are inserted into the HAproxy listening
+#   service configuration block.
+#   Defaults to []
+#
 # [*galera_bind_options*]
 #   (optional) A hash of options that are inserted into the HAproxy listening
 #   service configuration block.
@@ -450,7 +462,11 @@
 #
 # [*sensu_api_port*]
 #   (optional) Port of Sensu API service.
-#   Defaults to '4567'
+#   Defaults to '4568'
+#
+# [*redis_port*]
+#   (optional) Port of redis service.
+#   Defaults to '6379'
 #
 # [*vip_public_ip*]
 #  (optional) Array or string for public VIP
@@ -504,6 +520,7 @@ class cloud::loadbalancer(
   $kibana                           = true,
   $sensu_dashboard                  = true,
   $sensu_api                        = true,
+  $redis                            = true,
   $haproxy_auth                     = 'admin:changeme',
   $keepalived_state                 = 'BACKUP',
   $keepalived_priority              = '50',
@@ -539,6 +556,7 @@ class cloud::loadbalancer(
   $kibana_bind_options              = [],
   $sensu_dashboard_bind_options     = [],
   $sensu_api_bind_options           = [],
+  $redis_bind_options               = [],
   $ks_ceilometer_public_port        = 8777,
   $ks_cinder_public_port            = 8776,
   $ks_ec2_public_port               = 8773,
@@ -562,7 +580,8 @@ class cloud::loadbalancer(
   $elasticsearch_port               = 9200,
   $kibana_port                      = 8300,
   $sensu_dashboard_port             = 3000,
-  $sensu_api_port                   = 4567,
+  $sensu_api_port                   = 4568,
+  $redis_port                       = 6379,
   $vip_public_ip                    = ['127.0.0.1'],
   $vip_internal_ip                  = false,
   $vip_monitor_ip                   = false,
@@ -706,12 +725,19 @@ class cloud::loadbalancer(
     port              => $sensu_dashboard_port,
     bind_options      => $sensu_dashboard_bind_options,
     firewall_settings => $firewall_settings,
+    options           => {
+      'balance' => 'source',
+    },
   }
   cloud::loadbalancer::binding { 'sensu_api':
     ip                => $sensu_api,
     port              => $sensu_api_port,
     bind_options      => $sensu_api_bind_options,
     firewall_settings => $firewall_settings,
+    options           => {
+      'balance' => 'source',
+      'rspadd'  => ['Access-Control-Allow-Origin:\ *', 'Access-Control-Allow-Headers:\ origin,\ x-requested-with,\ content-type', 'Access-Control-Allow-Methods:\ PUT,\ GET,\ POST,\ DELETE,\ OPTIONS'],
+    },
   }
   cloud::loadbalancer::binding { 'spice_cluster':
     ip                => $spice,
@@ -883,6 +909,19 @@ class cloud::loadbalancer(
     firewall_settings => $firewall_settings,
   }
 
+  cloud::loadbalancer::binding { 'redis_cluster':
+    ip                => $redis,
+    port              => $redis_port,
+    options           => {
+      'mode'      => 'tcp',
+      'balance'   => 'first',
+      'option'    => ['tcp-check',],
+      'tcp-check' => ['send info\ replication\r\n','expect string role:master'],
+    },
+    bind_options      => $redis_bind_options,
+    firewall_settings => $firewall_settings,
+  }
+
   if (member(any2array($keepalived_public_ipvs), $galera_ip)) {
     warning('Exposing Galera cluster to public network is a security issue.')
   }

manifests/messaging.pp

@@ -102,11 +102,11 @@ class cloud::messaging(
     package_provider         => $package_provider,
   }
 
-  rabbitmq_vhost { '/':
+  rabbitmq_vhost { ['/', '/sensu']:
     provider => 'rabbitmqctl',
     require  => Class['rabbitmq'],
   }
-  rabbitmq_user { ['nova','glance','neutron','cinder','ceilometer','heat','trove']:
+  rabbitmq_user { ['nova','glance','neutron','cinder','ceilometer','heat','trove', 'sensu']:
     admin    => true,
     password => $rabbit_password,
     provider => 'rabbitmqctl',
@@ -120,6 +120,7 @@ class cloud::messaging(
     'ceilometer@/',
     'heat@/',
     'trove@/',
+    'sensu@/sensu',
   ]:
     configure_permission => '.*',
     write_permission     => '.*',

manifests/monitoring/server/sensu.pp

@@ -44,30 +44,18 @@
 #       }
 #     }
 #
-# [*manage_rabbitmq_resources*]
+# [*manage_sensu_plugins*]
-#   (optionnal) A boolean that determines if the RabbitMQ resources should be exported
+#   (optionnal) A boolean that determines if the Sensu plugins resources should be exported
 #   from this node
 #   Defaults to 'true'
 #
-# [*rabbitmq_user*]
-#   (optionnal) Rabbitmq user
-#   Defaults to 'sensu'
-#
-# [*rabbitmq_password*]
-#   (optionnal) Rabbitmq_password
-#   Defaults to 'rabbitpassword'
-#
-# [*rabbitmq_vhost*]
-#   (optionnal) Rabbitmq vhost
-#   Defaults to '/sensu'
-#
 # [*sensu_api_ip*]
 #   (optionnal) IP address to bind the sensu_api to
 #   Defaults to '%{::ipaddress}'
 #
 # [*sensu_api_port*]
 #   (optionnal) Port to bind the sensu_api to
-#   Defaults to '4567'
+#   Defaults to '4568'
 #
 # [*uchiwa_ip*]
 #   (optionnal) IP address to bind uchiwa to
@@ -86,12 +74,9 @@ class cloud::monitoring::server::sensu (
   $checks                    = {},
   $handlers                  = {},
   $plugins                   = {},
-  $manage_rabbitmq_resources = true,
+  $manage_sensu_plugins      = true,
-  $rabbitmq_user             = 'sensu',
-  $rabbitmq_password         = 'rabbitpassword',
-  $rabbitmq_vhost            = '/sensu',
   $sensu_api_ip              = $::ipaddress,
-  $sensu_api_port            = '4567',
+  $sensu_api_port            = '4568',
   $uchiwa_ip                 = $::ipaddress,
   $uchiwa_port               = '3000',
   $firewall_settings         = {},
@@ -99,67 +84,51 @@ class cloud::monitoring::server::sensu (
 
   include cloud::params
 
-  if $manage_rabbitmq_resources {
+  Service['sensu-api'] -> Service['uchiwa']
-    @@rabbitmq_user { $rabbitmq_user :
+  Service['sensu-server'] -> Service['uchiwa']
-      password => $rabbitmq_password,
+  Service['sensu-server'] -> Sensu::Plugin <<| |>>
+
+  include cloud::monitoring::agent::sensu
+
+  create_resources('sensu::check', $checks)
+  create_resources('sensu::handler', $handlers)
+
+  if $manage_sensu_plugins {
+    create_resources('@@sensu::plugin', $plugins)
+  }
+
+  include ::uchiwa
+  uchiwa::api { 'OpenStack' :
+    host => $sensu_api_ip,
+    port => $sensu_api_port,
+  }
+
+  if $::cloud::manage_firewall {
+    cloud::firewall::rule{ '100 allow sensu_dashboard access':
+      port   => $uchiwa_port,
+      extras => $firewall_settings,
     }
-    @@rabbitmq_vhost { $rabbitmq_vhost :
+
-      ensure  => present,
+    cloud::firewall::rule{ '100 allow sensu_api access':
-    }
+      port   => $sensu_api_port,
-    @@rabbitmq_user_permissions { "${rabbitmq_user}@${rabbitmq_vhost}" :
+      extras => $firewall_settings,
-      configure_permission => '.*',
-      read_permission      => '.*',
-      write_permission     => '.*',
     }
   }
 
-  $rabbitmq_user_realized = query_nodes("Rabbitmq_user['${rabbitmq_user}']")
+  @@haproxy::balancermember{"${::fqdn}-sensu_dashboard":
+    listening_service => 'sensu_dashboard',
+    server_names      => $::hostname,
+    ipaddresses       => $uchiwa_ip,
+    ports             => $uchiwa_port,
+    options           => 'check inter 2000 rise 2 fall 5'
+  }
 
-  if size($rabbitmq_user_realized) >= 1 {
+  @@haproxy::balancermember{"${::fqdn}-sensu_api":
-
+    listening_service => 'sensu_api',
-    Service["${::cloud::params::redis_service_name}"] -> Service['sensu-api'] -> Service['sensu-server'] -> Service['uchiwa']
+    server_names      => $::hostname,
-    Service['sensu-server'] -> Sensu::Plugin <<| |>>
+    ipaddresses       => $sensu_api_ip,
-
+    ports             => $sensu_api_port,
-
+    options           => 'check inter 2000 rise 2 fall 5'
-    include cloud::monitoring::agent::sensu
-
-    create_resources('sensu::check', $checks)
-    create_resources('sensu::handler', $handlers)
-    create_resources('@@sensu::plugin', $plugins)
-
-    include ::uchiwa
-    uchiwa::api { 'OpenStack' :
-      host => $uchiwa_ip,
-      port => $uchiwa_port,
-    }
-
-    if $::cloud::manage_firewall {
-      cloud::firewall::rule{ '100 allow sensu_dashboard access':
-        port   => $uchiwa_port,
-        extras => $firewall_settings,
-      }
-
-      cloud::firewall::rule{ '100 allow sensu_api access':
-        port   => $sensu_api_port,
-        extras => $firewall_settings,
-      }
-    }
-
-    @@haproxy::balancermember{"${::fqdn}-sensu_dashboard":
-      listening_service => 'sensu_dashoard',
-      server_names      => $::hostname,
-      ipaddresses       => $uchiwa_ip,
-      ports             => $uchiwa_port,
-      options           => 'check inter 2000 rise 2 fall 5'
-    }
-
-    @@haproxy::balancermember{"${::fqdn}-sensu_api":
-      listening_service => 'sensu_api',
-      server_names      => $::hostname,
-      ipaddresses       => $sensu_api_ip,
-      ports             => $sensu_api_port,
-      options           => 'check inter 2000 rise 2 fall 5'
-    }
   }
 
 }

spec/classes/cloud_database_nosql_redis_sentinel_spec.rb

@@ -28,6 +28,13 @@ describe 'cloud::database::nosql::redis::sentinel' do
 
     it { should create_class('redis::sentinel') }
 
+    it { should contain_file('/bin/redis-notifications.sh').with(
+      :ensure  => 'present',
+      :owner   => 'root',
+      :group   => 'root',
+      :mode    => '0755',
+    ) }
+
     context 'with default firewall enabled' do
       let :pre_condition do
         "class { 'cloud': manage_firewall => true }"

templates/database/redis-notifications.sh.erb

@@ -0,0 +1,30 @@
+#!/bin/bash
+HAPROXY="<%= @haproxy_monitor_ip %>:<%= @haproxy_monitor_port %>"
+CMD="$1"
+ARGS="$2"
+ARG1=`echo $ARGS | awk '{print $1}'`
+
+
+call_curl () {
+	DATA=`echo "s=$1&action=$2&b=%234" | sed -e s/:/%3A/`
+	curl --silent -o /dev/null $HAPROXY --data "$DATA"
+	echo curl $HAPROXY --data "$DATA"
+	return 0
+}
+
+
+[ "$CMD" = "+odown" ] && [ "$ARG1" = "master" ] && \
+	call_curl `echo $ARGS | awk '{print $2 ":" $3 ":" $4}'` 'disable'
+
+[ "$CMD" = "+sdown" ] && [ "$ARG1" = "slave" ] && \
+	call_curl `echo $ARGS | awk '{print $6 ":" $3 ":" $4}'` 'disable'
+
+[ "$CMD" = "+switch-master" ] && \
+	call_curl `echo $ARGS | awk '{print $1 ":" $4 ":" $5}'` 'enable' &&
+	call_curl `echo $ARGS | awk '{print $1 ":" $2 ":" $3}'` 'disable'
+
+[ "$CMD" = "-odown" ] && [ "$ARG1" = "master" ] && \
+	call_curl `echo $ARGS | awk '{print $2 ":" $3 ":" $4}'` 'enable'
+
+# without exit code, sentinel thinks the script is still running and locks any further execution
+exit 0