Refactor logstash_worker into log_processor module

Separate the jenkins log client and worker bits into a new module
called log_processor with ::client and ::worker classes.

Instantiate two workers on each logstash worker node.

Change-Id: I7cfec410983c25633e6b555f22a85e9435884cfb

modules/log_processor/manifests/client.pp

@@ -0,0 +1,63 @@
+# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# == Class: log_processor::client
+#
+class log_processor::client (
+  $config_file,
+) {
+
+  file { '/etc/logstash/jenkins-log-client.yaml':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0555',
+    source  => $config_file,
+    require => File['/etc/logstash'],
+  }
+
+  file { '/etc/init.d/jenkins-log-client':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0555',
+    source  => 'puppet:///modules/log_processor/jenkins-log-client.init',
+    require => [
+      File['/usr/local/bin/log-gearman-client.py'],
+      File['/etc/logstash/jenkins-log-client.yaml'],
+    ],
+  }
+
+  service { 'jenkins-log-client':
+    enable     => true,
+    hasrestart => true,
+    subscribe  => File['/etc/logstash/jenkins-log-client.yaml'],
+    require    => File['/etc/init.d/jenkins-log-client'],
+  }
+
+  include logrotate
+  logrotate::file { 'log-client-debug.log':
+    log     => '/var/log/logstash/log-client-debug.log',
+    options => [
+      'compress',
+      'copytruncate',
+      'missingok',
+      'rotate 7',
+      'daily',
+      'notifempty',
+    ],
+    require => Service['jenkins-log-client'],
+  }
+}

modules/log_processor/manifests/init.pp

@@ -0,0 +1,88 @@
+# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# == Class: log_processor
+#
+class log_processor (
+) {
+  package { 'python-daemon':
+    ensure => present,
+  }
+
+  package { 'python-zmq':
+    ensure => present,
+  }
+
+  package { 'python-yaml':
+    ensure => present,
+  }
+
+  package { 'crm114':
+    ensure => present,
+  }
+
+  include pip
+  package { 'gear':
+    ensure   => latest,
+    provider => 'pip',
+    require  => Class['pip'],
+  }
+
+  file { '/var/lib/crm114':
+    ensure  => directory,
+    owner   => 'logstash',
+    group   => 'logstash',
+    require => User['logstash'],
+  }
+
+  file { '/usr/local/bin/classify-log.crm':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    source  => 'puppet:///modules/log_processor/classify-log.crm',
+    require => [
+      Package['crm114'],
+    ],
+  }
+
+  file { '/usr/local/bin/log-gearman-client.py':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    source  => 'puppet:///modules/log_processor/log-gearman-client.py',
+    require => [
+      Package['python-daemon'],
+      Package['python-zmq'],
+      Package['python-yaml'],
+      Package['gear'],
+    ],
+  }
+
+  file { '/usr/local/bin/log-gearman-worker.py':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    source  => 'puppet:///modules/log_processor/log-gearman-worker.py',
+    require => [
+      Package['python-daemon'],
+      Package['python-zmq'],
+      Package['python-yaml'],
+      Package['gear'],
+    ],
+  }
+}

modules/log_processor/manifests/worker.pp

@@ -0,0 +1,67 @@
+# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# == Class: log_processor::worker
+#
+define log_processor::worker (
+  $config_file,
+) {
+  $suffix = "-${name}"
+
+  file { "/etc/logstash/jenkins-log-worker${suffix}.yaml":
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0555',
+    source  => $config_file,
+    require => Class['logstash::indexer'],
+  }
+
+  file { "/etc/init.d/jenkins-log-worker${suffix}":
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0555',
+    content => template('log_processor/jenkins-log-worker.init.erb'),
+    require => [
+      File['/usr/local/bin/log-gearman-worker.py'],
+      File["/etc/logstash/jenkins-log-worker${suffix}.yaml"],
+    ],
+  }
+
+  service { "jenkins-log-worker${suffix}":
+    enable     => true,
+    hasrestart => true,
+    subscribe  => File["/etc/logstash/jenkins-log-worker${suffix}.yaml"],
+    require    => [
+      Class['logstash::indexer'],
+      File["/etc/init.d/jenkins-log-worker${suffix}"],
+    ],
+  }
+
+  include logrotate
+  logrotate::file { "log-worker${suffix}-debug.log":
+    log     => "/var/log/logstash/log-worker${suffix}-debug.log",
+    options => [
+      'compress',
+      'copytruncate',
+      'missingok',
+      'rotate 7',
+      'daily',
+      'notifempty',
+    ],
+    require => Service["jenkins-log-worker${suffix}"],
+  }
+}

modules/log_processor/templates/jenkins-log-worker.init.erb

@@ -14,10 +14,10 @@
 # PATH should only include /usr/* if it runs after the mountnfs.sh script
 PATH=/sbin:/usr/sbin:/bin:/usr/bin
 DESC="Jenkins Log Worker"
-NAME=jenkins-log-worker
+NAME=jenkins-log-worker<%= suffix %>
 DAEMON=/usr/local/bin/log-gearman-worker.py
 PIDFILE=/var/run/$NAME/$NAME.pid
-DAEMON_ARGS="-c /etc/logstash/jenkins-log-worker.yaml -d /var/log/logstash/log-worker-debug.log -p $PIDFILE"
+DAEMON_ARGS="-c /etc/logstash/jenkins-log-worker<%= suffix %>.yaml -d /var/log/logstash/log-worker<%= suffix %>-debug.log -p $PIDFILE"
 SCRIPTNAME=/etc/init.d/$NAME
 USER=logstash
 

modules/openstack_project/manifests/logstash.pp

@@ -43,78 +43,10 @@ class openstack_project::logstash (
     proxy_elasticsearch => true,
   }
 
-  package { 'python-daemon':
-    ensure => present,
-  }
+  class { 'log_processor': }
 
-  package { 'python-zmq':
-    ensure => present,
-  }
-
-  package { 'python-yaml':
-    ensure => present,
-  }
-
-  include pip
-  package { 'gear':
-    ensure   => latest,
-    provider => 'pip',
-    require  => Class['pip'],
-  }
-
-  file { '/usr/local/bin/log-gearman-client.py':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0755',
-    source  => 'puppet:///modules/openstack_project/logstash/log-gearman-client.py',
-    require => [
-      Package['python-daemon'],
-      Package['python-zmq'],
-      Package['python-yaml'],
-      Package['gear'],
-    ],
-  }
-
-  file { '/etc/logstash/jenkins-log-client.yaml':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0555',
-    source  => 'puppet:///modules/openstack_project/logstash/jenkins-log-client.yaml',
-  }
-
-  file { '/etc/init.d/jenkins-log-client':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0555',
-    source  => 'puppet:///modules/openstack_project/logstash/jenkins-log-client.init',
-    require => [
-      File['/usr/local/bin/log-gearman-client.py'],
-      File['/etc/logstash/jenkins-log-client.yaml'],
-    ],
-  }
-
-  service { 'jenkins-log-client':
-    enable     => true,
-    hasrestart => true,
-    subscribe  => File['/etc/logstash/jenkins-log-client.yaml'],
-    require    => File['/etc/init.d/jenkins-log-client'],
-  }
-
-  include logrotate
-  logrotate::file { 'log-client-debug.log':
-    log     => '/var/log/logstash/log-client-debug.log',
-    options => [
-      'compress',
-      'copytruncate',
-      'missingok',
-      'rotate 7',
-      'daily',
-      'notifempty',
-    ],
-    require => Service['jenkins-log-client'],
+  class { 'log_processor::client':
+    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-client.yaml',
   }
 
   class { 'elastic_recheck::bot':

modules/openstack_project/manifests/logstash_worker.pp

@@ -31,102 +31,11 @@ class openstack_project::logstash_worker (
     conf_template => 'openstack_project/logstash/indexer.conf.erb',
   }
 
-  package { 'python-daemon':
-    ensure => present,
+  include log_processor
+  log_processor::worker { 'A':
+    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
   }
-
-  package { 'python-zmq':
-    ensure => present,
-  }
-
-  package { 'python-yaml':
-    ensure => present,
-  }
-
-  package { 'crm114':
-    ensure => present,
-  }
-
-  include pip
-  package { 'gear':
-    ensure   => latest,
-    provider => 'pip',
-    require  => Class['pip'],
-  }
-
-  file { '/var/lib/crm114':
-    ensure  => directory,
-    owner   => 'logstash',
-    group   => 'logstash',
-    require => User['logstash'],
-  }
-
-  file { '/usr/local/bin/classify-log.crm':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0755',
-    source  => 'puppet:///modules/openstack_project/logstash/classify-log.crm',
-    require => [
-      Package['crm114'],
-    ],
-  }
-
-  file { '/usr/local/bin/log-gearman-worker.py':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0755',
-    source  => 'puppet:///modules/openstack_project/logstash/log-gearman-worker.py',
-    require => [
-      Package['python-daemon'],
-      Package['python-zmq'],
-      Package['python-yaml'],
-      Package['gear'],
-    ],
-  }
-
-  file { '/etc/logstash/jenkins-log-worker.yaml':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0555',
-    source  => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
-  }
-
-  file { '/etc/init.d/jenkins-log-worker':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0555',
-    source  => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.init',
-    require => [
-      File['/usr/local/bin/log-gearman-worker.py'],
-      File['/etc/logstash/jenkins-log-worker.yaml'],
-    ],
-  }
-
-  service { 'jenkins-log-worker':
-    enable     => true,
-    hasrestart => true,
-    subscribe  => File['/etc/logstash/jenkins-log-worker.yaml'],
-    require    => [
-      Class['logstash::indexer'],
-      File['/etc/init.d/jenkins-log-worker'],
-    ],
-  }
-
-  include logrotate
-  logrotate::file { 'log-worker-debug.log':
-    log     => '/var/log/logstash/log-worker-debug.log',
-    options => [
-      'compress',
-      'copytruncate',
-      'missingok',
-      'rotate 7',
-      'daily',
-      'notifempty',
-    ],
-    require => Service['jenkins-log-worker'],
+  log_processor::worker { 'B':
+    config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
   }
 }