From 504852d528ef7b56060fa11388849ead912c3152 Mon Sep 17 00:00:00 2001 From: zhubingbing <1392607554@qq.com> Date: Mon, 10 Oct 2016 01:02:13 +0000 Subject: [PATCH] add grafana log in heka Change-Id: I38b57cc43411b703dbb0639a600aaef58e01bc38 Closes-Bug: #1631818 --- ansible/roles/common/tasks/config.yml | 2 + .../templates/cron-logrotate-grafana.conf.j2 | 3 + ansible/roles/common/templates/cron.json.j2 | 1 + .../common/templates/heka-grafana.toml.j2 | 18 +++++ ansible/roles/common/templates/heka.json.j2 | 7 ++ .../heka/plugins/decoders/os_grafana_log.lua | 72 +++++++++++++++++++ .../add-grafana-in-heka-6397498442c00670.yaml | 3 + 7 files changed, 106 insertions(+) create mode 100644 ansible/roles/common/templates/cron-logrotate-grafana.conf.j2 create mode 100644 ansible/roles/common/templates/heka-grafana.toml.j2 create mode 100644 docker/heka/plugins/decoders/os_grafana_log.lua create mode 100644 releasenotes/notes/add-grafana-in-heka-6397498442c00670.yaml diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index 426760502e..52e246f67a 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -30,6 +30,7 @@ - { name: "elasticsearch", enabled: "{{ enable_central_logging }}" } - { name: "global", enabled: "yes" } - { name: "gnocchi", enabled: "{{ enable_gnocchi }}" } + - { name: "grafana", enabled: "{{ enable_grafana }}" } - { name: "haproxy", enabled: "{{ enable_haproxy }}" } - { name: "horizon", enabled: "{{ enable_horizon }}" } - { name: "keepalived", enabled: "{{ enable_haproxy }}" } @@ -77,6 +78,7 @@ - { name: "glance", enabled: "{{ enable_glance }}" } - { name: "global", enabled: "yes" } - { name: "gnocchi", enabled: "{{ enable_gnocchi }}" } + - { name: "grafana", enabled: "{{ enable_grafana }}" } - { name: "haproxy", enabled: "{{ enable_haproxy }}" } - { name: "heat", enabled: "{{ enable_heat }}" } - { name: "keepalived", enabled: "{{ enable_haproxy }}" } diff --git a/ansible/roles/common/templates/cron-logrotate-grafana.conf.j2 b/ansible/roles/common/templates/cron-logrotate-grafana.conf.j2 new file mode 100644 index 0000000000..2a3a0889a4 --- /dev/null +++ b/ansible/roles/common/templates/cron-logrotate-grafana.conf.j2 @@ -0,0 +1,3 @@ +"/var/log/kolla/grafana/*.log" +{ +} diff --git a/ansible/roles/common/templates/cron.json.j2 b/ansible/roles/common/templates/cron.json.j2 index 7e1f4ab17a..c44529892c 100644 --- a/ansible/roles/common/templates/cron.json.j2 +++ b/ansible/roles/common/templates/cron.json.j2 @@ -7,6 +7,7 @@ ( 'cloudkitty', enable_cloudkitty ), ( 'glance', enable_glance ), ( 'gnocchi', enable_gnocchi ), + ( 'grafana', enable_grafana ), ( 'haproxy', enable_haproxy ), ( 'heat', enable_heat ), ( 'keepalived', enable_haproxy ), diff --git a/ansible/roles/common/templates/heka-grafana.toml.j2 b/ansible/roles/common/templates/heka-grafana.toml.j2 new file mode 100644 index 0000000000..7565c083e1 --- /dev/null +++ b/ansible/roles/common/templates/heka-grafana.toml.j2 @@ -0,0 +1,18 @@ +[grafana_log_decoder] +type = "SandboxDecoder" +filename = "lua_decoders/os_grafana_log.lua" + +[grafana_log_splitter] +type = "RegexSplitter" +delimiter = '\n\n(=[^=]+====)' +delimiter_eol = false +deliver_incomplete_final = true + +[grafana_logstreamer_input] +type = "LogstreamerInput" +decoder = "grafana_log_decoder" +splitter = "grafana_log_splitter" +log_directory = "/var/log/kolla" +file_match = 'grafana/(?Pgrafana.*)\.log\.?(?P\d*)$' +priority = ["^Seq"] +differentiator = ["Service"] diff --git a/ansible/roles/common/templates/heka.json.j2 b/ansible/roles/common/templates/heka.json.j2 index bb2ae35040..08aaedc96c 100644 --- a/ansible/roles/common/templates/heka.json.j2 +++ b/ansible/roles/common/templates/heka.json.j2 @@ -29,6 +29,13 @@ "perm": "0600", "optional": {{ (not enable_gnocchi | bool) | string | lower }} }, + { + "source": "{{ container_config_directory }}/heka-grafana.toml", + "dest": "/etc/heka/heka-grafana.toml", + "owner": "heka", + "perm": "0600", + "optional": {{ (not enable_grafana | bool) | string | lower }} + }, { "source": "{{ container_config_directory }}/heka-barbican.toml", "dest": "/etc/heka/heka-barbican.toml", diff --git a/docker/heka/plugins/decoders/os_grafana_log.lua b/docker/heka/plugins/decoders/os_grafana_log.lua new file mode 100644 index 0000000000..794145d6b3 --- /dev/null +++ b/docker/heka/plugins/decoders/os_grafana_log.lua @@ -0,0 +1,72 @@ +-- Copyright 2015-2016 Mirantis, Inc. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +local dt = require "date_time" +local l = require 'lpeg' +l.locale(l) + +local patt = require 'os_patterns' +local utils = require 'os_utils' + +local msg = { + Timestamp = nil, + Type = 'log', + Hostname = nil, + Payload = nil, + Pid = nil, + Fields = nil, + Severity = nil, +} + +-- Grafana message logs are formatted like this: +-- =ERROR REPORT==== 2-Jan-2015::09:17:22 === +-- Blabla +-- Blabla +-- +local message = l.Cg(patt.Message / utils.chomp, "Message") +-- The token before 'REPORT' isn't standardized so it can be a valid severity +-- level as 'INFO' or 'ERROR' but also 'CRASH' or 'SUPERVISOR'. +local severity = l.Cg(l.R"AZ"^1, "SeverityLabel") +local day = l.R"13" * l.R"09" + l.R"19" +local datetime = l.Cg(day, "day") * patt.dash * dt.date_mabbr * patt.dash * dt.date_fullyear * + "::" * dt.rfc3339_partial_time +local timestamp = l.Cg(l.Ct(datetime)/ dt.time_to_ns, "Timestamp") + +local grammar = l.Ct("=" * severity * " REPORT==== " * timestamp * " ===" * l.P'\n' * message) + +function process_message () + local log = read_message("Payload") + + local m = grammar:match(log) + if not m then + return -1 + end + + msg.Timestamp = m.Timestamp + msg.Payload = m.Message + msg.Logger = read_message("Logger") + + if utils.label_to_severity_map[m.SeverityLabel] then + msg.Severity = utils.label_to_severity_map[m.SeverityLabel] + elseif m.SeverityLabel == 'CRASH' then + msg.Severity = 2 -- CRITICAL + else + msg.Severity = 5 -- NOTICE + end + + msg.Fields = {} + msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity] + msg.Fields.programname = 'grafana' + + return utils.safe_inject_message(msg) +end diff --git a/releasenotes/notes/add-grafana-in-heka-6397498442c00670.yaml b/releasenotes/notes/add-grafana-in-heka-6397498442c00670.yaml new file mode 100644 index 0000000000..20c9b250d5 --- /dev/null +++ b/releasenotes/notes/add-grafana-in-heka-6397498442c00670.yaml @@ -0,0 +1,3 @@ +--- +features: + - add grafana log in heka