491aff0b88
This patch includes changes relative to integrating Heka with Elasticsearch and Kibana. The main change is the addition of an Heka ElasticSearchOutput plugin to make Heka send the logs it collects to Elasticsearch. Since Logstash is not used the enable_elk deploy variable is renamed to enable_central_logging. If enable_central_logging is false then Elasticsearch and Kibana are not started, and Heka won't attempt to send logs to Elasticsearch. By default enable_central_logging is set to false. If enable_central_logging is set to true after deployment then the Heka container needs to be recreated (for Heka to get the new configuration). The Kibana configuration used property names that are deprecated in Kibana 4.2. This is changed to use non-deprecated property names. Previously logs read from files and from Syslog had a different Type in Heka. This is changed to always use "log" for the Type. In this way just one index instead of two is used in Elasticsearch, making things easier to the user on the visualization side. The HAProxy configuration is changed to add entries for Kibana. Kibana server is now accessible via the internal VIP, and also via the external VIP if there's one configured. The HAProxy configuration is changed to add an entry for Elasticsearch. So Elasticsearch is now accessible via the internal VIP. Heka uses that channel for communicating with Elasticsearch. Note that currently the Heka logs include "Plugin elasticsearch_output" errors when Heka starts. This occurs when Heka starts processing logs while Elasticsearch is not yet started. These are transient errors that go away when Elasticsearch is ready. And with buffering enabled on the ElasticSearchOuput plugin logs will be buffered and then retransmitted when Elasticsearch is ready. Change-Id: I6ff7a4f0ad04c4c666e174693a35ff49914280bb Implements: blueprint central-logging-service
56 lines
1.7 KiB
Lua
56 lines
1.7 KiB
Lua
-- Copyright 2016 Mirantis, Inc.
|
|
--
|
|
-- Licensed under the Apache License, Version 2.0 (the "License");
|
|
-- you may not use this file except in compliance with the License.
|
|
-- You may obtain a copy of the License at
|
|
--
|
|
-- http://www.apache.org/licenses/LICENSE-2.0
|
|
--
|
|
-- Unless required by applicable law or agreed to in writing, software
|
|
-- distributed under the License is distributed on an "AS IS" BASIS,
|
|
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
-- See the License for the specific language governing permissions and
|
|
-- limitations under the License.
|
|
--
|
|
-- The code in this file was inspired by Heka's rsyslog.lua decoder plugin.
|
|
-- https://github.com/mozilla-services/heka/blob/master/sandbox/lua/decoders/rsyslog.lua
|
|
|
|
local syslog = require "syslog"
|
|
local utils = require "os_utils"
|
|
|
|
local msg = {
|
|
Timestamp = nil,
|
|
Type = 'log',
|
|
Hostname = read_config("hostname"),
|
|
Payload = nil,
|
|
Pid = nil,
|
|
Severity = nil,
|
|
Fields = nil
|
|
}
|
|
|
|
-- See https://tools.ietf.org/html/rfc3164
|
|
local grammar = syslog.build_rsyslog_grammar('<%PRI%>%TIMESTAMP% %syslogtag% %msg%')
|
|
|
|
function process_message ()
|
|
local log = read_message("Payload")
|
|
local fields = grammar:match(log)
|
|
if not fields then return -1 end
|
|
|
|
msg.Timestamp = fields.timestamp
|
|
fields.timestamp = nil
|
|
|
|
msg.Severity = fields.pri.severity
|
|
fields.syslogfacility = fields.pri.facility
|
|
fields.pri = nil
|
|
|
|
fields.programname = fields.syslogtag.programname
|
|
msg.Pid = fields.syslogtag.pid
|
|
fields.syslogtag = nil
|
|
|
|
msg.Payload = fields.msg
|
|
fields.msg = nil
|
|
|
|
msg.Fields = fields
|
|
return utils.safe_inject_message(msg)
|
|
end
|