diff --git a/ansible/roles/common/templates/heka-global.toml.j2 b/ansible/roles/common/templates/heka-global.toml.j2 index c9fbc1e7ec..5e325026ae 100644 --- a/ansible/roles/common/templates/heka-global.toml.j2 +++ b/ansible/roles/common/templates/heka-global.toml.j2 @@ -11,9 +11,20 @@ filename = "lua_decoders/os_syslog.lua" [syslog_log_decoder.config] hostname = "{{ ansible_hostname }}" +[swift_log_decoder] +type = "SandboxDecoder" +filename = "lua_decoders/os_swift_log.lua" + [swift_log_decoder.config] + hostname = "{{ ansible_hostname }}" + +[multi_log_decoder] +type = "MultiDecoder" +subs = ["syslog_log_decoder", "swift_log_decoder"] +cascade_strategy = "first-wins" + [syslog_input] type = "UdpInput" net = "unixgram" address = "/var/lib/kolla/heka/log" -decoder = "syslog_log_decoder" +decoder = "multi_log_decoder" splitter = "NullSplitter" diff --git a/ansible/roles/common/templates/heka.json.j2 b/ansible/roles/common/templates/heka.json.j2 index 30ed998804..34471008d5 100644 --- a/ansible/roles/common/templates/heka.json.j2 +++ b/ansible/roles/common/templates/heka.json.j2 @@ -46,7 +46,7 @@ "perm": "0600" } {%- if not loop.last %},{% endif %} -{% endfor %} +{% endfor -%} ] } diff --git a/ansible/roles/swift/templates/account.conf.j2 b/ansible/roles/swift/templates/account.conf.j2 index d3c2f26441..ad54582cc7 100644 --- a/ansible/roles/swift/templates/account.conf.j2 +++ b/ansible/roles/swift/templates/account.conf.j2 @@ -6,7 +6,7 @@ mount_check = false log_address = /var/lib/kolla/heka/log log_name = {{ service_name }} log_facility = LOG_LOCAL0 -log_level = warning +log_level = INFO [pipeline:main] pipeline = account-server diff --git a/ansible/roles/swift/templates/container.conf.j2 b/ansible/roles/swift/templates/container.conf.j2 index 879dd9c3c9..55e0976dd0 100644 --- a/ansible/roles/swift/templates/container.conf.j2 +++ b/ansible/roles/swift/templates/container.conf.j2 @@ -6,7 +6,7 @@ mount_check = false log_address = /var/lib/kolla/heka/log log_name = {{ service_name }} log_facility = LOG_LOCAL0 -log_level = warning +log_level = INFO [pipeline:main] pipeline = container-server diff --git a/ansible/roles/swift/templates/object.conf.j2 b/ansible/roles/swift/templates/object.conf.j2 index b3856228cf..7499bbfd2f 100644 --- a/ansible/roles/swift/templates/object.conf.j2 +++ b/ansible/roles/swift/templates/object.conf.j2 @@ -7,7 +7,7 @@ mount_check = false log_address = /var/lib/kolla/heka/log log_name = {{ service_name }} log_facility = LOG_LOCAL0 -log_level = warning +log_level = INFO [pipeline:main] {% if service_name == 'swift-object-expirer' %} diff --git a/ansible/roles/swift/templates/proxy-server.conf.j2 b/ansible/roles/swift/templates/proxy-server.conf.j2 index e8ea502e26..cf9de196fa 100644 --- a/ansible/roles/swift/templates/proxy-server.conf.j2 +++ b/ansible/roles/swift/templates/proxy-server.conf.j2 @@ -5,7 +5,7 @@ bind_port = {{ swift_proxy_server_port }} log_address = /var/lib/kolla/heka/log log_name = {{ service_name }} log_facility = LOG_LOCAL0 -log_level = warning +log_level = INFO [pipeline:main] pipeline = catch_errors gatekeeper healthcheck cache container_sync bulk ratelimit authtoken keystoneauth slo dlo proxy-server diff --git a/docker/heka/plugins/decoders/os_swift_log.lua b/docker/heka/plugins/decoders/os_swift_log.lua new file mode 100644 index 0000000000..440e9ab08b --- /dev/null +++ b/docker/heka/plugins/decoders/os_swift_log.lua @@ -0,0 +1,49 @@ +-- Copyright 2016 Mirantis, Inc. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- The code in this file was inspired by Heka's rsyslog.lua decoder plugin. +-- https://github.com/mozilla-services/heka/blob/master/sandbox/lua/decoders/rsyslog.lua + +local syslog = require "syslog" +local utils = require "os_utils" + +local msg = { + Timestamp = nil, + Type = 'Syslog', + Hostname = read_config("hostname"), + Payload = nil, + Pid = nil, + Severity = nil, + Fields = nil +} + +-- See https://github.com/openstack/swift/blob/2a8b455/swift/common/utils.py#L1423-L1424 +local swift_grammar = syslog.build_rsyslog_grammar('<%PRI%>%programname%: %msg%') + +function process_message () + local log = read_message("Payload") + + local fields = swift_grammar:match(log) + if not fields then return -1 end + + msg.Severity = fields.pri.severity + fields.syslogfacility = fields.pri.facility + fields.pri = nil + + msg.Payload = fields.msg + fields.msg = nil + + msg.Fields = fields + return utils.safe_inject_message(msg) +end