Merge "Enable mqtt notifications from logstash gearman worker"

modules/openstack_project/files/logstash/jenkins-log-worker.yaml

@@ -1,7 +0,0 @@
-gearman-host: logstash.openstack.org
-gearman-port: 4730
-output-host: localhost
-output-port: 9999
-output-mode: tcp
-crm114-script: /usr/local/bin/classify-log.crm
-crm114-data: /var/lib/crm114

modules/openstack_project/manifests/logstash_worker.pp

@@ -18,7 +18,7 @@ class openstack_project::logstash_worker (
   $discover_node = 'elasticsearch01.openstack.org',
   $filter_rev    = 'master',
   $filter_source = 'https://git.openstack.org/openstack-infra/logstash-filters',
-  $enable_mqtt = false,
+  $enable_mqtt = true,
   $mqtt_hostname = 'firehose.openstack.org',
   $mqtt_port = 8883,
   $mqtt_topic = "logstash/${::hostname}",
@@ -26,6 +26,15 @@ class openstack_project::logstash_worker (
   $mqtt_password = undef,
   $mqtt_ca_cert_contents = undef,
 ) {
+
+  file { '/etc/logprocessor/worker.yaml':
+    ensure => present,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    content => template('openstack_project/logstash/jenkins-log-worker.yaml.erb'),
+  }
+
   file { '/etc/default/logstash-indexer':
     ensure => present,
     owner  => 'root',
@@ -53,26 +62,33 @@ class openstack_project::logstash_worker (
     notify  => Service['logstash'],
   }
 
+  file { '/etc/logstash/mqtt-root-CA.pem.crt':
+      ensure  => present,
+      content => $mqtt_ca_cert_contents,
+      replace => true,
+      owner   => 'subunit',
+      group   => 'subunit',
+      mode    => '0555',
+  }
+
   validate_array($elasticsearch_nodes)  # needed by output.conf.erb
   class { '::logstash::indexer':
     input_template         => 'openstack_project/logstash/input.conf.erb',
     output_template        => 'openstack_project/logstash/output.conf.erb',
-    enable_mqtt            => $enable_mqtt,
-    mqtt_ca_cert_contents  => $mqtt_ca_cert_contents,
     require                => Logstash::Filter['openstack-logstash-filters'],
   }
 
   include ::log_processor
   log_processor::worker { 'A':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
   log_processor::worker { 'B':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
   log_processor::worker { 'C':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
   log_processor::worker { 'D':
-    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
+    config_file => '/etc/logprocessor/worker.yaml',
   }
 }

modules/openstack_project/templates/logstash/jenkins-log-worker.yaml.erb

@@ -0,0 +1,13 @@
+gearman-host: logstash.openstack.org
+gearman-port: 4730
+output-host: localhost
+output-port: 9999
+output-mode: tcp
+crm114-script: /usr/local/bin/classify-log.crm
+crm114-data: /var/lib/crm114
+mqtt-host: <%= @mqtt_hostname %>
+mqtt-port: <%= @mqtt_port %>
+mqtt-topic: gearman-logstash/<%= @hostname %>
+mqtt-user: <%= @mqtt_user %>
+mqtt-pass: <%= @mqtt_pass %>
+mqtt-ca-certs: /etc/logstash/mqtt-root-CA.pem.crt

modules/openstack_project/templates/logstash/output.conf.erb

@@ -4,16 +4,5 @@ output {
     manage_template => false
     flush_size => 1024
   }
-<% if @enable_mqtt -%>
-  mqtt {
-    host => "<%= @mqtt_hostname %>"
-    port => <%= @mqtt_port %>
-    topic => "<%= @mqtt_topic %>"
-    username => "<%= @mqtt_username %>"
-    password => "<%= @mqtt_password %>"
-    ssl => true
-    ca_file => "/etc/logstash/mqtt-root-CA.pem.crt"
-  }
-<% end -%>
 
 }