Add UI option to disable logs and notifications indexation

This change also enables the collection of logs and notifications even
if the events indexation is disabled. The reason is that useful metrics
are extracted from this data such as HTTP response times, resource
creation time and so on.
The term 'event' is used to reference both logs and OpenStack notifcations.

Change-Id: I28814129f745a8ef3ee44d3d3cf1cc267337b840

deployment_scripts/puppet/manifests/base.pp

@@ -40,7 +40,30 @@ if hiera('deployment_mode') =~ /^ha_/ and $is_controller {
   $additional_groups = []
 }
 
-if $is_controller and $lma_collector['enable_notifications'] {
+$elasticsearch_mode = $lma_collector['elasticsearch_mode']
+
+case $elasticsearch_mode {
+  'remote': {
+    $es_server = $lma_collector['elasticsearch_address']
+  }
+  'local': {
+    $es_node_name = $lma_collector['elasticsearch_node_name']
+    $es_nodes = filter_nodes(hiera('nodes'), 'user_node_name', $es_node_name)
+    if size($es_nodes) < 1 {
+      fail("Could not find node '${es_node_name}' in the environment")
+    }
+    $es_server = $es_nodes[0]['internal_address']
+  }
+  'disabled': {
+    # Nothing to do
+  }
+  default: {
+    fail("'${elasticsearch_mode}' mode not supported for Elasticsearch")
+  }
+}
+
+# Notifications are always collected even when event indexation is disabled
+if $is_controller{
   $pre_script        = '/usr/local/bin/wait_for_rabbitmq'
   # Params used by the script.
   $rabbit            = hiera('rabbit')
@@ -66,19 +89,26 @@ class { 'lma_collector':
   pre_script => $pre_script,
 }
 
-class { 'lma_collector::logs::system':
-  require => Class['lma_collector'],
-}
-
-if (str2bool($::ovs_log_directory)){
-  # install logstreamer for open vSwitch if log directory exists
-  class { 'lma_collector::logs::ovs':
+if $elasticsearch_mode != 'disabled' {
+  class { 'lma_collector::logs::system':
     require => Class['lma_collector'],
   }
-}
 
-class { 'lma_collector::logs::monitor':
-  require => Class['lma_collector'],
+  if (str2bool($::ovs_log_directory)){
+    # install logstreamer for open vSwitch if log directory exists
+    class { 'lma_collector::logs::ovs':
+      require => Class['lma_collector'],
+    }
+  }
+
+  class { 'lma_collector::logs::monitor':
+    require => Class['lma_collector'],
+  }
+
+  class { 'lma_collector::elasticsearch':
+    server  => $es_server,
+    require => Class['lma_collector'],
+  }
 }
 
 $influxdb_mode = $lma_collector['influxdb_mode']
@@ -126,26 +156,3 @@ case $influxdb_mode {
     fail("'${influxdb_mode}' mode not supported for InfluxDB")
   }
 }
-
-$elasticsearch_mode = $lma_collector['elasticsearch_mode']
-case $elasticsearch_mode {
-  'remote': {
-    $es_server = $lma_collector['elasticsearch_address']
-  }
-  'local': {
-    $es_node_name = $lma_collector['elasticsearch_node_name']
-    $es_nodes = filter_nodes(hiera('nodes'), 'user_node_name', $es_node_name)
-    if size($es_nodes) < 1 {
-      fail("Could not find node '${es_node_name}' in the environment")
-    }
-    $es_server = $es_nodes[0]['internal_address']
-  }
-  default: {
-    fail("'${elasticsearch_mode}' mode not supported for Elasticsearch")
-  }
-}
-
-class { 'lma_collector::elasticsearch':
-  server  => $es_server,
-  require => Class['lma_collector'],
-}

deployment_scripts/puppet/manifests/cinder.pp

@@ -17,7 +17,11 @@ include lma_collector::params
 $ceilometer    = hiera('ceilometer')
 $lma_collector = hiera('lma_collector')
 
-$enable_notifications = $lma_collector['enable_notifications']
+if $lma_collector['elasticsearch_mode'] != 'disabled' {
+
+  class { 'lma_collector::logs::openstack': }
+}
+
 if $ceilometer['enabled'] {
   $notification_topics = [$lma_collector::params::openstack_topic, $lma_collector::params::lma_topic]
 }
@@ -25,10 +29,7 @@ else {
   $notification_topics = [$lma_collector::params::lma_topic]
 }
 
-class { 'lma_collector::logs::openstack': }
-
-if $enable_notifications {
-  class { 'lma_collector::notifications::cinder':
-    topics  => $notification_topics,
-  }
+# OpenStack notifcations are always useful for indexation and metrics collection
+class { 'lma_collector::notifications::cinder':
+  topics  => $notification_topics,
 }

deployment_scripts/puppet/manifests/compute.pp

@@ -17,7 +17,13 @@ include lma_collector::params
 $ceilometer    = hiera('ceilometer')
 $lma_collector = hiera('lma_collector')
 
-$enable_notifications = $lma_collector['enable_notifications']
+if $lma_collector['elasticsearch_mode'] != 'disabled' {
+
+  class { 'lma_collector::logs::openstack': }
+
+  class { 'lma_collector::logs::libvirt': }
+}
+
 if $ceilometer['enabled'] {
   $notification_topics = [$lma_collector::params::openstack_topic, $lma_collector::params::lma_topic]
 }
@@ -25,12 +31,7 @@ else {
   $notification_topics = [$lma_collector::params::lma_topic]
 }
 
-class { 'lma_collector::logs::openstack': }
-
-class { 'lma_collector::logs::libvirt': }
-
-if $enable_notifications {
-  class { 'lma_collector::notifications::compute':
-    topics  => $notification_topics,
-  }
+# OpenStack notifcations are always useful for indexation and metrics collection
+class { 'lma_collector::notifications::compute':
+  topics  => $notification_topics,
 }

deployment_scripts/puppet/manifests/controller.pp

@@ -16,13 +16,8 @@ include lma_collector::params
 
 $ceilometer     = hiera('ceilometer')
 $lma_collector  = hiera('lma_collector')
-$management_vip = hiera('management_vip')
-$nova           = hiera('nova')
-$cinder         = hiera('cinder')
 $rabbit         = hiera('rabbit')
-$neutron        = hiera('quantum_settings')
 
-$enable_notifications = $lma_collector['enable_notifications']
 if $ceilometer['enabled'] {
   $notification_topics = [$lma_collector::params::openstack_topic, $lma_collector::params::lma_topic]
 }
@@ -43,38 +38,46 @@ if hiera('deployment_mode') =~ /^ha_/ {
   $ha_deployment = false
 }
 
-if $ha_deployment {
-  $rabbitmq_pid_file = '/var/run/rabbitmq/p_pid'
-}
-else {
-  $rabbitmq_pid_file = '/var/run/rabbitmq/pid'
+# OpenStack notifcations are always useful for indexation and metrics collection
+class { 'lma_collector::notifications::controller':
+  host     => '127.0.0.1',
+  port     => hiera('amqp_port', '5673'),
+  user     => $rabbitmq_user,
+  password => $rabbit['password'],
+  topics   => $notification_topics,
 }
 
-# Logs
+# OpenStack logs are always useful for indexation and metrics collection
 class { 'lma_collector::logs::openstack': }
 
-class { 'lma_collector::logs::mysql': }
+# Logs
+if $lma_collector['elasticsearch_mode'] != 'disabled' {
 
-class { 'lma_collector::logs::rabbitmq': }
+  class { 'lma_collector::logs::mysql': }
 
-if $ha_deployment {
-  class { 'lma_collector::logs::pacemaker': }
-}
+  class { 'lma_collector::logs::rabbitmq': }
 
-# Notifications
-if $enable_notifications {
-  class { 'lma_collector::notifications::controller':
-    host     => '127.0.0.1',
-    port     => hiera('amqp_port', '5673'),
-    user     => $rabbitmq_user,
-    password => $rabbit['password'],
-    topics   => $notification_topics,
+  if $ha_deployment {
+    class { 'lma_collector::logs::pacemaker': }
   }
+
 }
 
 # Metrics
 if $lma_collector['influxdb_mode'] != 'disabled' {
 
+  $nova           = hiera('nova')
+  $neutron        = hiera('quantum_settings')
+  $cinder         = hiera('cinder')
+  $management_vip = hiera('management_vip')
+
+  if $ha_deployment {
+    $rabbitmq_pid_file = '/var/run/rabbitmq/p_pid'
+  }
+  else {
+    $rabbitmq_pid_file = '/var/run/rabbitmq/pid'
+  }
+
   if $ha_deployment {
     $haproxy_socket = '/var/lib/haproxy/stats'
   }else{
@@ -145,9 +148,8 @@ if $lma_collector['influxdb_mode'] != 'disabled' {
 
   class { 'lma_collector::logs::metrics': }
 
-  if $enable_notifications {
-    class { 'lma_collector::notifications::metrics': }
-  }
+  # Notification are always collected, lets extract metrics from there
+  class { 'lma_collector::notifications::metrics': }
 
   # Enable Apache status module
   class { 'lma_collector::mod_status': }

environment_config.yaml

@@ -15,8 +15,10 @@ attributes:
     type: "radio"
     weight: 20
     value: "local"
-    label: "Logging analytics"
+    label: "Events analytics (logs and notifications)"
     values:
+      - data: "disabled"
+        label: "Disabled"
       - data: "local"
         label: "Local node"
       - data: "remote"
@@ -45,17 +47,11 @@ attributes:
       source: '^[a-zA-Z\d][a-zA-Z\d_\-.]+$'
       error: "Invalid address or name"
 
-  enable_notifications:
-    type: "checkbox"
-    weight: 50
-    value: false
-    label: "Collect OpenStack notifications"
-
   influxdb_mode:
     type: "radio"
     weight: 60
-    value: "disabled"
-    label: "Metric analytics"
+    value: "local"
+    label: "Metrics analytics"
     values:
       - data: "disabled"
         label: "Disabled"
@@ -97,6 +93,7 @@ attributes:
     restrictions: &disable_influxdb_parameters
         - condition: "settings:lma_collector.influxdb_mode.value != 'remote'"
           action: "disable"
+
   influxdb_user:
     value: 'lma'
     label: 'InfluxDB user'