x/fuel-plugin-lma-collector
Code Issues Proposed changes

Initial import of the LMA collector plugin

Browse Source 
This is an import of the initial LMA PoC code. For now, it only covers
the collection of logs (notifications will be added in a subsequent
commit).

There's been a bit of rewrite to:
- decouple the Heka configuration from the LMA collector.
- run the Heka service as non-root when possible (Ubuntu only for now
  due to file permission issues on CentOS [1]).
- adapt to version 0.9 of Heka.

[1] https://bugs.launchpad.net/fuel/+bug/1425954

Change-Id: I4472b49a25e18e06984b5b29bdce18f917137bc8
This commit is contained in:
Simon Pasquier
2015-02-24 15:47:28 +01:00
parent 6854302bd6
commit c9ee4d30d9
58 changed files with 2443 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
deployment_scripts/puppet/modules/lma_collector/manifests/logs/monitor.pp Normal file
View File

@@ -0,0 +1,11 @@
class lma_collector::logs::monitor {
include lma_collector::params
heka::filter::sandbox { 'log_monitor':
config_dir => $lma_collector::params::config_dir,
filename => "${lma_collector::plugins_dir}/filters/log_monitor.lua" ,
message_matcher => "Type == 'log'",
ticker_interval => 60,
notify => Service[$lma_collector::params::service_name],
}
}

21
deployment_scripts/puppet/modules/lma_collector/manifests/logs/mysql.pp Normal file
View File

@@ -0,0 +1,21 @@
class lma_collector::logs::mysql {
include lma_collector::params
heka::decoder::sandbox { 'mysql':
config_dir => $lma_collector::params::config_dir,
filename => "${lma_collector::plugins_dir}/decoders/mysql_log.lua" ,
config => {
syslog_pattern => $lma_collector::params::syslog_pattern
},
notify => Service[$lma_collector::params::service_name],
}
heka::input::logstreamer { 'mysql':
config_dir => $lma_collector::params::config_dir,
decoder => 'mysql',
file_match => 'mysqld\.log$',
differentiator => "['mysql']",
require => Heka::Decoder::Sandbox['mysql'],
notify => Service[$lma_collector::params::service_name],
}
}

41
deployment_scripts/puppet/modules/lma_collector/manifests/logs/openstack.pp Normal file
View File

@@ -0,0 +1,41 @@
class lma_collector::logs::openstack {
include lma_collector::params
heka::decoder::sandbox { 'openstack':
config_dir => $lma_collector::params::config_dir,
filename => "${lma_collector::plugins_dir}/decoders/openstack_log.lua" ,
config => {
syslog_pattern => $lma_collector::params::syslog_pattern
},
notify => Service[$lma_collector::params::service_name],
}
# Use the <PRI> token as the delimiter because OpenStack services may log
# messages with newlines and the configuration of the Syslog daemon doesn't
# escape them.
heka::splitter::regex { 'openstack':
config_dir => $lma_collector::params::config_dir,
delimiter => '(<[0-9]+>)',
delimiter_eol => false,
notify => Service[$lma_collector::params::service_name],
}
heka::input::logstreamer { 'openstack':
config_dir => $lma_collector::params::config_dir,
decoder => 'openstack',
splitter => 'openstack',
file_match => '(?P<Service>nova|cinder|keystone|glance|heat|neutron)-all\.log$',
differentiator => "[ 'openstack.', 'Service' ]",
require => [Heka::Decoder::Sandbox['openstack'], Heka::Splitter::Regex['openstack']],
notify => Service[$lma_collector::params::service_name],
}
heka::input::logstreamer { 'openstack_dashboard':
config_dir => $lma_collector::params::config_dir,
decoder => 'openstack',
file_match => 'dashboard\.log$',
differentiator => "[ 'openstack.horizon' ]",
require => Heka::Decoder::Sandbox['openstack'],
notify => Service[$lma_collector::params::service_name],
}
}

32
deployment_scripts/puppet/modules/lma_collector/manifests/logs/pacemaker.pp Normal file
View File

@@ -0,0 +1,32 @@
class lma_collector::logs::pacemaker {
include lma_collector::params
heka::decoder::sandbox { 'pacemaker':
config_dir => $lma_collector::params::config_dir,
filename => "${lma_collector::plugins_dir}/decoders/generic_syslog.lua" ,
config => {
syslog_pattern => $lma_collector::params::syslog_pattern
},
notify => Service[$lma_collector::params::service_name],
}
# Use the <PRI> token as the delimiter because Pacemaker may log messages
# with newlines and the configuration of the Syslog daemon doesn't escape
# them.
heka::splitter::regex { 'pacemaker':
config_dir => $lma_collector::params::config_dir,
delimiter => '(<[0-9]+>)',
delimiter_eol => false,
notify => Service[$lma_collector::params::service_name],
}
heka::input::logstreamer { 'pacemaker':
config_dir => $lma_collector::params::config_dir,
decoder => 'pacemaker',
splitter => 'pacemaker',
file_match => 'pacemaker\.log$',
differentiator => "[ 'pacemaker' ]",
require => [Heka::Decoder::Sandbox['pacemaker'], Heka::Splitter::Regex['pacemaker']],
notify => Service[$lma_collector::params::service_name],
}
}

27
deployment_scripts/puppet/modules/lma_collector/manifests/logs/rabbitmq.pp Normal file
View File

@@ -0,0 +1,27 @@
class lma_collector::logs::rabbitmq {
include lma_collector::params
heka::decoder::sandbox { 'rabbitmq':
config_dir => $lma_collector::params::config_dir,
filename => "${lma_collector::plugins_dir}/decoders/rabbitmq.lua" ,
notify => Service[$lma_collector::params::service_name],
}
heka::splitter::regex { 'rabbitmq':
config_dir => $lma_collector::params::config_dir,
delimiter => '\n(=[^=]+====)',
delimiter_eol => false,
notify => Service[$lma_collector::params::service_name],
}
heka::input::logstreamer { 'rabbitmq':
config_dir => $lma_collector::params::config_dir,
decoder => 'rabbitmq',
splitter => 'rabbitmq',
log_directory => "/var/log/rabbitmq",
file_match => 'rabbit@(?P<Node>.+)\.log$',
differentiator => '["rabbitmq.", "Node"]',
require => [Heka::Decoder::Sandbox['rabbitmq'], Heka::Splitter::Regex['rabbitmq']],
notify => Service[$lma_collector::params::service_name],
}
}

21
deployment_scripts/puppet/modules/lma_collector/manifests/logs/system.pp Normal file
View File

@@ -0,0 +1,21 @@
class lma_collector::logs::system {
include lma_collector::params
heka::decoder::sandbox { 'system':
config_dir => $lma_collector::params::config_dir,
filename => "${lma_collector::plugins_dir}/decoders/generic_syslog.lua" ,
config => {
syslog_pattern => $lma_collector::params::syslog_pattern
},
notify => Service[$lma_collector::params::service_name],
}
heka::input::logstreamer { 'system':
config_dir => $lma_collector::params::config_dir,
decoder => 'system',
file_match => '(?P<Service>daemon|cron|kern|auth)\.log$',
differentiator => "[ 'system.', 'Service' ]",
require => Heka::Decoder::Sandbox['system'],
notify => Service[$lma_collector::params::service_name],
}
}