ebac150f8a
This change separates the processing of the logs/notifications and metric/alerting into 2 dedicated hekad processes, these services are named 'log_collector' and 'metric_collector'. Both services are managed by Pacemaker on controller nodes and by Upstart on other nodes. All metrics computed by log_collector (HTTP response times and creation time for instances and volumes) are sent directly to the metric_collector via TCP. Elasticsearch output (log_collector) uses full_action='block' and the TCP output uses full_action='drop'. All outputs of metric_collector (InfluxDB, HTTP and TCP) use full_action='drop'. The buffer size configurations are: * metric_collector: - influxdb-output buffer size is increased to 1Gb. - aggregator-output (tcp) buffer size is decreased to 256Mb (vs 1Gb). - nagios outputs (x3) buffer size are decreased to 1Mb. * log_collector: - elasticsearch-output buffer size is decreased to 256Mb (vs 1Gb). - tcp-output buffer size is set to 256Mb. Implements: blueprint separate-lma-collector-pipelines Fixes-bug: #1566748 Change-Id: Ieadb93b89f81e944e21cf8e5a65f4d683fd0ffb8
80 lines
2.8 KiB
Puppet
80 lines
2.8 KiB
Puppet
# Copyright 2015 Mirantis, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
|
|
#
|
|
# == Class lma_collector::logs::swift
|
|
#
|
|
# Class that configures Heka for reading Swift logs.
|
|
#
|
|
# The following rsyslog pattern is assumed:
|
|
#
|
|
# <%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%\n
|
|
#
|
|
# Swift only uses syslog and doesn't add its logs to log files located in a
|
|
# /var/log/swift/ directory as other OpenStack services do. So we make Swift a
|
|
# special case and assume rsyslog is used.
|
|
#
|
|
# === Parameters:
|
|
#
|
|
# [*file_match*]
|
|
# (mandatory) The log file name pattern. Example: 'swift\.log$'. Example
|
|
# for a sequential rotating logfile: 'swift\.log\.?(?P<Seq>\d*)$'. See
|
|
# http://hekad.readthedocs.org/en/latest/pluginconfig/logstreamer.html
|
|
# for more information.
|
|
#
|
|
# [*priority*]
|
|
# (optional) When using sequential logstreams, the priority defines how
|
|
# to sort the logfiles in order from the slowest to newest. Example:
|
|
# '["^Seq"]'. See
|
|
# http://hekad.readthedocs.org/en/latest/pluginconfig/logstreamer.html
|
|
# for more information.
|
|
#
|
|
# [*log_directory*]
|
|
# (optional) The log directory. Default is /var/log.
|
|
#
|
|
class lma_collector::logs::swift (
|
|
$file_match,
|
|
$priority = undef,
|
|
$log_directory = $lma_collector::params::log_directory,
|
|
) inherits lma_collector::params {
|
|
include lma_collector::service::log
|
|
|
|
$lua_modules_dir = $lma_collector::params::lua_modules_dir
|
|
|
|
$config_dir = $lma_collector::params::log_config_dir
|
|
# Note: syslog_pattern could be made configurable in the future.
|
|
heka::decoder::sandbox { 'swift':
|
|
config_dir => $config_dir,
|
|
filename => "${lma_collector::params::plugins_dir}/decoders/generic_syslog.lua",
|
|
config => {
|
|
syslog_pattern => $lma_collector::params::syslog_pattern,
|
|
fallback_syslog_pattern => $lma_collector::params::fallback_syslog_pattern
|
|
},
|
|
module_directory => $lua_modules_dir,
|
|
notify => Class['lma_collector::service::log'],
|
|
}
|
|
|
|
heka::input::logstreamer { 'swift':
|
|
config_dir => $config_dir,
|
|
decoder => 'swift',
|
|
log_directory => $log_directory,
|
|
file_match => $file_match,
|
|
differentiator => '[\'openstack.swift\']',
|
|
priority => $priority,
|
|
require => Heka::Decoder::Sandbox['swift'],
|
|
notify => Class['lma_collector::service::log'],
|
|
}
|
|
}
|