openstack
/
kolla
Code Issues Proposed changes

Remove heka image 

Heka is removed and replaced by fluentd.

Change-Id: I4ed0f61b02bea5dcb02b61a9eb99e3ff3f460682
This commit is contained in:
Jeffrey Zhang 2017-02-24 19:13:12 +08:00
parent 4a15181f3f
commit b9d07a04da
21 changed files with 0 additions and 1284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.rst
View File

@ -129,8 +129,6 @@ Kolla provides images to deploy the following infrastructure components:
- `HAProxy <http://www.haproxy.org/>`__ and
`Keepalived <http://www.keepalived.org/>`__ for high availability of services
and their endpoints.
- `Heka <http://hekad.readthedocs.org/>`__ A distributed and
scalable logging system for OpenStack services.
- `Kafka <http://kafka.apache.org/documentation/>`__ A distributed streaming
platform.
- `MariaDB and Galera Cluster <https://mariadb.com/kb/en/mariadb/galera-cluster/>`__

38
docker/heka/Dockerfile.j2
View File

@ -1,38 +0,0 @@
FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
MAINTAINER {{ maintainer }}
{% block heka_header %}{% endblock %}
{% import "macros.j2" as macros with context %}
{{ macros.configure_user(name='heka', homedir='/var/cache/hekad') }}
{% if base_distro in ['centos', 'oraclelinux', 'rhel'] %}
{% set heka_packages = [
'https://github.com/mozilla-services/heka/releases/download/v0.10.0/heka-0_10_0-linux-amd64.rpm'
] %}
RUN mkdir /etc/heka
{% elif base_distro in ['ubuntu', 'debian'] %}
{% set heka_packages = [
'https://github.com/mozilla-services/heka/releases/download/v0.10.0/heka_0.10.0_amd64.deb'
] %}
{% endif %}
{{ macros.install_packages(heka_packages | customizable("packages")) }}
COPY plugins/modules /usr/share/heka/lua_modules/
COPY plugins/decoders /usr/share/heka/lua_decoders/
COPY plugins/encoders /usr/share/heka/lua_encoders/
COPY heka_sudoers /etc/sudoers.d/kolla_heka_sudoers
COPY extend_start.sh /usr/local/bin/kolla_extend_start
RUN chmod 440 /etc/sudoers.d/kolla_heka_sudoers \
&& chmod 755 /usr/local/bin/kolla_extend_start \
&& chown -R heka: /usr/share/heka /etc/heka
{% block heka_footer %}{% endblock %}
{% block footer %}{% endblock %}
USER heka

37
docker/heka/extend_start.sh
View File

@ -1,37 +0,0 @@
#!/bin/bash
# Give processes executed with the "kolla" group the permission to create files
# and sub-directories in the /var/log/kolla directory.
#
# Also set the setgid permission on the /var/log/kolla directory so that new
# files and sub-directories in that directory inherit its group id ("kolla").
if [[ $(stat -c %U:%G /var/log/kolla) != "heka:kolla" ]]; then
sudo chown heka:kolla /var/log/kolla
fi
if [[ $(stat -c %a /var/log/kolla) != "2775" ]]; then
sudo chmod 2775 /var/log/kolla
fi
# Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
# of the KOLLA_BOOTSTRAP variable being set, including empty.
if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
exit 0
fi
# /var/cache/hekad is what the Heka daemon will use for persistent storage
# through process restarts, so make "heka" the user of that directory.
if [[ $(stat -c %U:%G /var/cache/hekad) != "heka:heka" ]]; then
sudo chown heka: /var/cache/hekad
fi
# Give hekad the permission to create the "log" Unix socket file in the
# /var/lib/kolla/heka directory.
if [[ $(stat -c %U:%G /var/lib/kolla/heka) != "heka:kolla" ]]; then
sudo chown heka:kolla /var/lib/kolla/heka
fi
# The Heka daemon will create the /var/lib/kolla/heka/log Unix socket file, so
# make sure it's not present or the Heka daemon will fail when starting.
if [[ -e "/var/lib/kolla/heka/log" ]]; then
rm -rf /var/lib/kolla/heka/log
fi

4
docker/heka/heka_sudoers
View File

@ -1,4 +0,0 @@
%kolla ALL=(root) NOPASSWD: /bin/chown heka\:kolla /var/log/kolla, /usr/bin/chown heka\:kolla /var/log/kolla
%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla
%kolla ALL=(root) NOPASSWD: /bin/chown heka\: /var/cache/hekad, /usr/bin/chown heka\: /var/cache/hekad
%kolla ALL=(root) NOPASSWD: /bin/chown heka\:kolla /var/lib/kolla/heka, /usr/bin/chown heka\:kolla /var/lib/kolla/heka

71
docker/heka/plugins/decoders/os_aodh_apache_log.lua
View File

@ -1,71 +0,0 @@
-- Copyright 2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local l = require 'lpeg'
l.locale(l)
local common_log_format = require 'common_log_format'
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = nil,
Severity = 6,
}
local severity_label = utils.severity_to_label_map[msg.Severity]
local apache_log_pattern = read_config("apache_log_pattern") or error(
"apache_log_pattern configuration must be specificed")
local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
local request_grammar = l.Ct(patt.http_request)
function process_message ()
local logger = read_message("Logger")
local log = read_message("Payload")
local m
m = apache_grammar:match(log)
if m then
msg.Logger = 'openstack.aodh'
msg.Payload = log
msg.Timestamp = m.time
msg.Fields = {}
msg.Fields.http_status = m.status
msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
msg.Fields.programname = logger
msg.Fields.severity_label = severity_label
local request = m.request
m = request_grammar:match(request)
if m then
msg.Fields.http_method = m.http_method
msg.Fields.http_url = m.http_url
msg.Fields.http_version = m.http_version
end
return utils.safe_inject_message(msg)
end
return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
end

72
docker/heka/plugins/decoders/os_ceilometer_apache_log.lua
View File

@ -1,72 +0,0 @@
-- Copyright 2015 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local l = require 'lpeg'
l.locale(l)
local common_log_format = require 'common_log_format'
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = nil,
Severity = 6,
}
local severity_label = utils.severity_to_label_map[msg.Severity]
local apache_log_pattern = read_config("apache_log_pattern") or error(
"apache_log_pattern configuration must be specificed")
local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
local request_grammar = l.Ct(patt.http_request)
function process_message ()
-- logger is either "ceilometer-apache-public" or "ceilometer-apache-admin"
local logger = read_message("Logger")
local log = read_message("Payload")
local m
m = apache_grammar:match(log)
if m then
msg.Logger = 'openstack.ceilometer'
msg.Payload = log
msg.Timestamp = m.time
msg.Fields = {}
msg.Fields.http_status = m.status
msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
msg.Fields.programname = logger
msg.Fields.severity_label = severity_label
local request = m.request
m = request_grammar:match(request)
if m then
msg.Fields.http_method = m.http_method
msg.Fields.http_url = m.http_url
msg.Fields.http_version = m.http_version
end
return utils.safe_inject_message(msg)
end
return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
end

71
docker/heka/plugins/decoders/os_gnocchi_apache_log.lua
View File

@ -1,71 +0,0 @@
-- Copyright 2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local l = require 'lpeg'
l.locale(l)
local common_log_format = require 'common_log_format'
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = nil,
Severity = 6,
}
local severity_label = utils.severity_to_label_map[msg.Severity]
local apache_log_pattern = read_config("apache_log_pattern") or error(
"apache_log_pattern configuration must be specificed")
local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
local request_grammar = l.Ct(patt.http_request)
function process_message ()
local logger = read_message("Logger")
local log = read_message("Payload")
local m
m = apache_grammar:match(log)
if m then
msg.Logger = 'openstack.gnocchi'
msg.Payload = log
msg.Timestamp = m.time
msg.Fields = {}
msg.Fields.http_status = m.status
msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
msg.Fields.programname = logger
msg.Fields.severity_label = severity_label
local request = m.request
m = request_grammar:match(request)
if m then
msg.Fields.http_method = m.http_method
msg.Fields.http_url = m.http_url
msg.Fields.http_version = m.http_version
end
return utils.safe_inject_message(msg)
end
return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
end

72
docker/heka/plugins/decoders/os_grafana_log.lua
View File

@ -1,72 +0,0 @@
-- Copyright 2015-2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local dt = require "date_time"
local l = require 'lpeg'
l.locale(l)
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = nil,
Severity = nil,
}
-- Grafana message logs are formatted like this:
-- =ERROR REPORT==== 2-Jan-2015::09:17:22 ===
-- Blabla
-- Blabla
--
local message = l.Cg(patt.Message / utils.chomp, "Message")
-- The token before 'REPORT' isn't standardized so it can be a valid severity
-- level as 'INFO' or 'ERROR' but also 'CRASH' or 'SUPERVISOR'.
local severity = l.Cg(l.R"AZ"^1, "SeverityLabel")
local day = l.R"13" * l.R"09" + l.R"19"
local datetime = l.Cg(day, "day") * patt.dash * dt.date_mabbr * patt.dash * dt.date_fullyear *
"::" * dt.rfc3339_partial_time
local timestamp = l.Cg(l.Ct(datetime)/ dt.time_to_ns, "Timestamp")
local grammar = l.Ct("=" * severity * " REPORT==== " * timestamp * " ===" * l.P'\n' * message)
function process_message ()
local log = read_message("Payload")
local m = grammar:match(log)
if not m then
return -1
end
msg.Timestamp = m.Timestamp
msg.Payload = m.Message
msg.Logger = read_message("Logger")
if utils.label_to_severity_map[m.SeverityLabel] then
msg.Severity = utils.label_to_severity_map[m.SeverityLabel]
elseif m.SeverityLabel == 'CRASH' then
msg.Severity = 2 -- CRITICAL
else
msg.Severity = 5 -- NOTICE
end
msg.Fields = {}
msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
msg.Fields.programname = 'grafana'
return utils.safe_inject_message(msg)
end

72
docker/heka/plugins/decoders/os_horizon_apache_log.lua
View File

@ -1,72 +0,0 @@
-- Copyright 2015 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local l = require 'lpeg'
l.locale(l)
local common_log_format = require 'common_log_format'
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = nil,
Severity = 6,
}
local severity_label = utils.severity_to_label_map[msg.Severity]
local apache_log_pattern = read_config("apache_log_pattern") or error(
"apache_log_pattern configuration must be specificed")
local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
local request_grammar = l.Ct(patt.http_request)
function process_message ()
-- logger is either "horizon-apache-public" or "horizon-apache-admin"
local logger = read_message("Logger")
local log = read_message("Payload")
local m
m = apache_grammar:match(log)
if m then
msg.Logger = 'openstack.horizon'
msg.Payload = log
msg.Timestamp = m.time
msg.Fields = {}
msg.Fields.http_status = m.status
msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
msg.Fields.programname = logger
msg.Fields.severity_label = severity_label
local request = m.request
m = request_grammar:match(request)
if m then
msg.Fields.http_method = m.http_method
msg.Fields.http_url = m.http_url
msg.Fields.http_version = m.http_version
end
return utils.safe_inject_message(msg)
end
return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
end

72
docker/heka/plugins/decoders/os_keystone_apache_log.lua
View File

@ -1,72 +0,0 @@
-- Copyright 2015 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local l = require 'lpeg'
l.locale(l)
local common_log_format = require 'common_log_format'
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = nil,
Severity = 6,
}
local severity_label = utils.severity_to_label_map[msg.Severity]
local apache_log_pattern = read_config("apache_log_pattern") or error(
"apache_log_pattern configuration must be specificed")
local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
local request_grammar = l.Ct(patt.http_request)
function process_message ()
-- logger is either "keystone-apache-public" or "keystone-apache-admin"
local logger = read_message("Logger")
local log = read_message("Payload")
local m
m = apache_grammar:match(log)
if m then
msg.Logger = 'openstack.keystone'
msg.Payload = log
msg.Timestamp = m.time
msg.Fields = {}
msg.Fields.http_status = m.status
msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
msg.Fields.programname = logger
msg.Fields.severity_label = severity_label
local request = m.request
m = request_grammar:match(request)
if m then
msg.Fields.http_method = m.http_method
msg.Fields.http_url = m.http_url
msg.Fields.http_version = m.http_version
end
return utils.safe_inject_message(msg)
end
return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
end

50
docker/heka/plugins/decoders/os_mysql_log.lua
View File

@ -1,50 +0,0 @@
-- Copyright 2015-2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
require "string"
local l = require 'lpeg'
l.locale(l)
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = {},
Severity = nil,
}
local sp = patt.sp
local colon = patt.colon
-- mysqld logs are cranky,the date is YYMMMDD, the hours have no leading zero and the "real" severity level is enclosed by square brackets...
local mysql_grammar = l.Ct(l.digit^-6 * sp^1 * l.digit^-2 * colon * l.digit^-2 * colon * l.digit^-2 * sp^1 * l.P"[" * l.Cg(l.R("az", "AZ")^0 / string.upper, "SeverityLabel") * l.P"]" * sp^1 * l.Cg(patt.Message, "Message"))
function process_message ()
local log = read_message("Payload")
local logger = read_message("Logger")
local m = mysql_grammar:match(log)
if not m then return -1 end
msg.Logger = logger
msg.Payload = m.Message
msg.Fields.severity_label = m.SeverityLabel
return utils.safe_inject_message(msg)
end

140
docker/heka/plugins/decoders/os_openstack_log.lua
View File

@ -1,140 +0,0 @@
-- Copyright 2015-2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
require "string"
require "table"
local l = require 'lpeg'
l.locale(l)
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = nil,
Severity = nil,
}
-- traceback_lines is a reference to a table used to accumulate lines of
-- a Traceback. traceback_key represent the key of the Traceback lines
-- being accumulated in traceback_lines. This is used to know when to
-- stop accumulating and inject the Heka message.
local traceback_key = nil
local traceback_lines = nil
function prepare_message (service, timestamp, pid, severity_label,
python_module, programname, payload)
msg.Logger = 'openstack.' .. service
msg.Timestamp = timestamp
msg.Payload = payload
msg.Pid = pid
msg.Severity = utils.label_to_severity_map[severity_label] or 7
msg.Fields = {}
msg.Fields.severity_label = severity_label
msg.Fields.python_module = python_module
msg.Fields.programname = programname
msg.Payload = payload
end
-- OpenStack log messages are of this form:
-- 2015-11-30 08:38:59.306 3434 INFO oslo_service.periodic_task [-] Blabla...
--
-- [-] is the "request" part, it can take multiple forms.
function process_message ()
-- Logger is of form "<service>_<program>" (e.g. "nova_nova-api",
-- "neutron_l3-agent").
local logger = read_message("Logger")
local service, program = string.match(logger, '([^_]+)_(.+)')
local log = read_message("Payload")
local m
m = patt.openstack:match(log)
if not m then
return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
end
local key = {
Timestamp = m.Timestamp,
Pid = m.Pid,
SeverityLabel = m.SeverityLabel,
PythonModule = m.PythonModule,
service = service,
program = program,
}
if traceback_key ~= nil then
-- If traceback_key is not nil then it means we've started accumulated
-- lines of a Python traceback. We keep accumulating the traceback
-- lines util we get a different log key.
if utils.table_equal(traceback_key, key) then
table.insert(traceback_lines, m.Message)
return 0
else
prepare_message(traceback_key.service, traceback_key.Timestamp,
traceback_key.Pid, traceback_key.SeverityLabel,
traceback_key.PythonModule, traceback_key.program,
table.concat(traceback_lines, ''))
traceback_key = nil
traceback_lines = nil
-- Ignore safe_inject_message status code here to still get a
-- chance to inject the current log message.
utils.safe_inject_message(msg)
end
end
if patt.traceback:match(m.Message) then
-- Python traceback detected, begin accumulating the lines making
-- up the traceback.
traceback_key = key
traceback_lines = {}
table.insert(traceback_lines, m.Message)
return 0
end
prepare_message(service, m.Timestamp, m.Pid, m.SeverityLabel, m.PythonModule,
program, m.Message)
m = patt.openstack_request_context:match(msg.Payload)
if m then
msg.Fields.request_id = m.RequestId
if m.UserId then
msg.Fields.user_id = m.UserId
end
if m.TenantId then
msg.Fields.tenant_id = m.TenantId
end
end
m = patt.openstack_http:match(msg.Payload)
if m then
msg.Fields.http_method = m.http_method
msg.Fields.http_status = m.http_status
msg.Fields.http_url = m.http_url
msg.Fields.http_version = m.http_version
msg.Fields.http_response_size = m.http_response_size
msg.Fields.http_response_time = m.http_response_time
m = patt.ip_address:match(msg.Payload)
if m then
msg.Fields.http_client_ip_address = m.ip_address
end
end
return utils.safe_inject_message(msg)
end

72
docker/heka/plugins/decoders/os_panko_apache_log.lua
View File

@ -1,72 +0,0 @@
-- Copyright 2015 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local l = require 'lpeg'
l.locale(l)
local common_log_format = require 'common_log_format'
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = nil,
Severity = 6,
}
local severity_label = utils.severity_to_label_map[msg.Severity]
local apache_log_pattern = read_config("apache_log_pattern") or error(
"apache_log_pattern configuration must be specificed")
local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
local request_grammar = l.Ct(patt.http_request)
function process_message ()
-- logger is either "panko-apache-public" or "panko-apache-admin"
local logger = read_message("Logger")
local log = read_message("Payload")
local m
m = apache_grammar:match(log)
if m then
msg.Logger = 'openstack.panko'
msg.Payload = log
msg.Timestamp = m.time
msg.Fields = {}
msg.Fields.http_status = m.status
msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
msg.Fields.programname = logger
msg.Fields.severity_label = severity_label
local request = m.request
m = request_grammar:match(request)
if m then
msg.Fields.http_method = m.http_method
msg.Fields.http_url = m.http_url
msg.Fields.http_version = m.http_version
end
return utils.safe_inject_message(msg)
end
return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
end

72
docker/heka/plugins/decoders/os_rabbitmq_log.lua
View File

@ -1,72 +0,0 @@
-- Copyright 2015-2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local dt = require "date_time"
local l = require 'lpeg'
l.locale(l)
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = nil,
Severity = nil,
}
-- RabbitMQ message logs are formatted like this:
-- =ERROR REPORT==== 2-Jan-2015::09:17:22 ===
-- Blabla
-- Blabla
--
local message = l.Cg(patt.Message / utils.chomp, "Message")
-- The token before 'REPORT' isn't standardized so it can be a valid severity
-- level as 'INFO' or 'ERROR' but also 'CRASH' or 'SUPERVISOR'.
local severity = l.Cg(l.R"AZ"^1, "SeverityLabel")
local day = l.R"13" * l.R"09" + l.R"19"
local datetime = l.Cg(day, "day") * patt.dash * dt.date_mabbr * patt.dash * dt.date_fullyear *
"::" * dt.rfc3339_partial_time
local timestamp = l.Cg(l.Ct(datetime)/ dt.time_to_ns, "Timestamp")
local grammar = l.Ct("=" * severity * " REPORT==== " * timestamp * " ===" * l.P'\n' * message)
function process_message ()
local log = read_message("Payload")
local m = grammar:match(log)
if not m then
return -1
end
msg.Timestamp = m.Timestamp
msg.Payload = m.Message
msg.Logger = read_message("Logger")
if utils.label_to_severity_map[m.SeverityLabel] then
msg.Severity = utils.label_to_severity_map[m.SeverityLabel]
elseif m.SeverityLabel == 'CRASH' then
msg.Severity = 2 -- CRITICAL
else
msg.Severity = 5 -- NOTICE
end
msg.Fields = {}
msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
msg.Fields.programname = 'rabbitmq'
return utils.safe_inject_message(msg)
end

49
docker/heka/plugins/decoders/os_swift_log.lua
View File

@ -1,49 +0,0 @@
-- Copyright 2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-- The code in this file was inspired by Heka's rsyslog.lua decoder plugin.
-- https://github.com/mozilla-services/heka/blob/master/sandbox/lua/decoders/rsyslog.lua
local syslog = require "syslog"
local utils = require "os_utils"
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = read_config("hostname"),
Payload = nil,
Pid = nil,
Severity = nil,
Fields = nil
}
-- See https://git.openstack.org/cgit/openstack/swift/tree/swift/common/utils.py?id=2a8b455#n1423
local swift_grammar = syslog.build_rsyslog_grammar('<%PRI%>%programname%: %msg%')
function process_message ()
local log = read_message("Payload")
local fields = swift_grammar:match(log)
if not fields then return -1 end
msg.Severity = fields.pri.severity
fields.syslogfacility = fields.pri.facility
fields.pri = nil
msg.Payload = fields.msg
fields.msg = nil
msg.Fields = fields
return utils.safe_inject_message(msg)
end

55
docker/heka/plugins/decoders/os_syslog.lua
View File

@ -1,55 +0,0 @@
-- Copyright 2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-- The code in this file was inspired by Heka's rsyslog.lua decoder plugin.
-- https://github.com/mozilla-services/heka/blob/master/sandbox/lua/decoders/rsyslog.lua
local syslog = require "syslog"
local utils = require "os_utils"
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = read_config("hostname"),
Payload = nil,
Pid = nil,
Severity = nil,
Fields = nil
}
-- See https://tools.ietf.org/html/rfc3164
local grammar = syslog.build_rsyslog_grammar('<%PRI%>%TIMESTAMP% %syslogtag% %msg%')
function process_message ()
local log = read_message("Payload")
local fields = grammar:match(log)
if not fields then return -1 end
msg.Timestamp = fields.timestamp
fields.timestamp = nil
msg.Severity = fields.pri.severity
fields.syslogfacility = fields.pri.facility
fields.pri = nil
fields.programname = fields.syslogtag.programname
msg.Pid = fields.syslogtag.pid
fields.syslogtag = nil
msg.Payload = fields.msg
fields.msg = nil
msg.Fields = fields
return utils.safe_inject_message(msg)
end

72
docker/heka/plugins/decoders/os_zaqar_apache_log.lua
View File

@ -1,72 +0,0 @@
-- Copyright 2015 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local l = require 'lpeg'
l.locale(l)
local common_log_format = require 'common_log_format'
local patt = require 'os_patterns'
local utils = require 'os_utils'
local msg = {
Timestamp = nil,
Type = 'log',
Hostname = nil,
Payload = nil,
Pid = nil,
Fields = nil,
Severity = 6,
}
local severity_label = utils.severity_to_label_map[msg.Severity]
local apache_log_pattern = read_config("apache_log_pattern") or error(
"apache_log_pattern configuration must be specificed")
local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
local request_grammar = l.Ct(patt.http_request)
function process_message ()
-- logger is either "zaqar-apache-public" or "zaqar-apache-admin"
local logger = read_message("Logger")
local log = read_message("Payload")
local m
m = apache_grammar:match(log)
if m then
msg.Logger = 'openstack.zaqar'
msg.Payload = log
msg.Timestamp = m.time
msg.Fields = {}
msg.Fields.http_status = m.status
msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
msg.Fields.programname = logger
msg.Fields.severity_label = severity_label
local request = m.request
m = request_grammar:match(request)
if m then
msg.Fields.http_method = m.http_method
msg.Fields.http_url = m.http_url
msg.Fields.http_version = m.http_version
end
return utils.safe_inject_message(msg)
end
return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
end

26
docker/heka/plugins/encoders/os_syslog.lua
View File

@ -1,26 +0,0 @@
-- Copyright 2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
require "string"
local interpolate = require "msg_interpolate"
local utils = require "os_utils"
local header_template = "<%{Severity}>%{%FT%TZ} %{Hostname} %{programname}[%{Pid}]:"
function process_message()
local timestamp = read_message("Timestamp") / 1e9
local header = interpolate.interpolate_from_msg(header_template, timestamp)
local payload = string.format("%s %s\n", header, read_message("Payload"))
return utils.safe_inject_payload("txt", "", payload)
end

144
docker/heka/plugins/modules/os_patterns.lua
View File

@ -1,144 +0,0 @@
-- Copyright 2015-2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local table = require 'table'
local dt = require "date_time"
local l = require 'lpeg'
l.locale(l)
local tonumber = tonumber
local M = {}
setfenv(1, M) -- Remove external access to contain everything in the module
function format_uuid(t)
return table.concat(t, '-')
end
function anywhere (patt)
return l.P {
patt + 1 * l.V(1)
}
end
sp = l.space
colon = l.P":"
dash = l.P"-"
dot = l.P'.'
quote = l.P'"'
local x4digit = l.xdigit * l.xdigit * l.xdigit * l.xdigit
local uuid_dash = l.C(x4digit * x4digit * dash * x4digit * dash * x4digit * dash * x4digit * dash * x4digit * x4digit * x4digit)
local uuid_nodash = l.Ct(l.C(x4digit * x4digit) * l.C(x4digit) * l.C(x4digit) * l.C(x4digit) * l.C(x4digit * x4digit * x4digit)) / format_uuid
-- Return a UUID string in canonical format (eg with dashes)
Uuid = uuid_nodash + uuid_dash
-- Parse a datetime string and return a table with the following keys
-- year (string)
-- month (string)
-- day (string)
-- hour (string)
-- min (string)
-- sec (string)
-- sec_frac (number less than 1, can be nil)
-- offset_sign ('-' or '+', can be nil)
-- offset_hour (number, can be nil)
-- offset_min (number, can be nil)
--
-- The datetime string can be formatted as
-- 'YYYY-MM-DD( |T)HH:MM:SS(.ssssss)?(offset indicator)?'
TimestampTable = l.Ct(dt.rfc3339_full_date * (sp + l.P"T") * dt.rfc3339_partial_time * (dt.rfc3339_time_offset + dt.timezone_offset)^-1)
-- Returns the parsed datetime converted to nanosec
Timestamp = TimestampTable / dt.time_to_ns
programname = (l.R("az", "AZ", "09") + l.P"." + dash + l.P"_")^1
Pid = l.digit^1
SeverityLabel = l.P"CRITICAL" + l.P"ERROR" + l.P"WARNING" + l.P"INFO" + l.P"AUDIT" + l.P"DEBUG"
Message = l.P(1)^0
-- Capture for OpenStack logs producing four values: Timestamp, Pid,
-- SeverityLabel, PythonModule and Message.
--
-- OpenStack log messages are of this form:
-- 2015-11-30 08:38:59.306 3434 INFO oslo_service.periodic_task [-] Blabla...
--
-- [-] is the "request" part, it can take multiple forms. See below.
openstack = l.Ct(l.Cg(Timestamp, "Timestamp")* sp * l.Cg(Pid, "Pid") * sp *
l.Cg(SeverityLabel, "SeverityLabel") * sp * l.Cg(programname, "PythonModule") *
sp * l.Cg(Message, "Message"))
-- Capture for OpenStack request context producing three values: RequestId,
-- UserId and TenantId.
--
-- Notes:
--
-- OpenStack logs include a request context, enclosed between square brackets.
-- It takes one of these forms:
--
-- [-]
-- [req-0fd2a9ba-448d-40f5-995e-33e32ac5a6ba - - - - -]
-- [req-4db318af-54c9-466d-b365-fe17fe4adeed 8206d40abcc3452d8a9c1ea629b4a8d0 112245730b1f4858ab62e3673e1ee9e2 - - -]
--
-- In the 1st case the capture produces nil.
-- In the 2nd case the capture produces one value: RequestId.
-- In the 3rd case the capture produces three values: RequestId, UserId, TenantId.
--
-- The request id may be formatted as 'req-xxx' or 'xxx' depending on the project.
-- The user id and tenant id may not be present depending on the OpenStack release.
openstack_request_context = (l.P(1) - "[" )^0 * "[" * l.P"req-"^-1 *
l.Ct(l.Cg(Uuid, "RequestId") * sp * ((l.Cg(Uuid, "UserId") * sp *
l.Cg(Uuid, "TenantId")) + l.P(1)^0)) - "]"
local http_method = l.Cg(l.R"AZ"^3, "http_method")
local url = l.Cg( (1 - sp)^1, "http_url")
local http_version = l.Cg(l.digit * dot * l.digit, "http_version")
-- Pattern for the "<http_method> <http_url> HTTP/<http_version>" format found
-- found in both OpenStack and Apache log files.
-- Example : OPTIONS /example.com HTTP/1.0
http_request = http_method * sp * url * sp * l.P'HTTP/' * http_version
-- Patterns for HTTP status, HTTP response size and HTTP response time in
-- OpenLayers logs.
--
-- Notes:
-- Nova changes the default log format of eventlet.wsgi (see nova/wsgi.py) and
-- prefixes the HTTP status, response size and response time values with
-- respectively "status: ", "len: " and "time: ".
-- Other OpenStack services just rely on the default log format.
-- TODO(pasquier-s): build the LPEG grammar based on the log_format parameter
-- passed to eventlet.wsgi.server similar to what the build_rsyslog_grammar
-- function does for RSyslog.
local openstack_http_status = l.P"status: "^-1 * l.Cg(l.digit^3, "http_status")
local openstack_response_size = l.P"len: "^-1 * l.Cg(l.digit^1 / tonumber, "http_response_size")
local openstack_response_time = l.P"time: "^-1 * l.Cg(l.digit^1 * dot^0 * l.digit^0 / tonumber, "http_response_time")
-- Capture for OpenStack HTTP producing six values: http_method, http_url,
-- http_version, http_status, http_response_size and http_response_time.
openstack_http = anywhere(l.Ct(
quote * http_request * quote * sp *
openstack_http_status * sp * openstack_response_size * sp *
openstack_response_time
))
-- Capture for IP addresses producing one value: ip_address.
ip_address = anywhere(l.Ct(
l.Cg(l.digit^-3 * dot * l.digit^-3 * dot * l.digit^-3 * dot * l.digit^-3, "ip_address")
))
-- Pattern used to match the beginning of a Python Traceback.
traceback = l.P'Traceback (most recent call last):'
return M

89
docker/heka/plugins/modules/os_utils.lua
View File

@ -1,89 +0,0 @@
-- Copyright 2015-2016 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local cjson = require 'cjson'
local string = require 'string'
local patt = require 'os_patterns'
local pairs = pairs
local inject_message = inject_message
local inject_payload = inject_payload
local read_message = read_message
local pcall = pcall
local M = {}
setfenv(1, M) -- Remove external access to contain everything in the module
severity_to_label_map = {
[0] = 'EMERGENCY',
[1] = 'ALERT',
[2] = 'CRITICAL',
[3] = 'ERROR',
[4] = 'WARNING',
[5] = 'NOTICE',
[6] = 'INFO',
[7] = 'DEBUG',
}
label_to_severity_map = {
EMERGENCY = 0,
ALERT = 1,
CRITICAL = 2,
ERROR = 3,
WARNING = 4,
NOTICE = 5,
INFO= 6,
DEBUG = 7,
}
function chomp(s)
return string.gsub(s, "\n$", "")
end
-- Call inject_message() wrapped by pcall()
function safe_inject_message(msg)
local ok, err_msg = pcall(inject_message, msg)
if not ok then
return -1, err_msg
else
return 0
end
end
-- Call inject_payload() wrapped by pcall()
function safe_inject_payload(payload_type, payload_name, data)
local ok, err_msg = pcall(inject_payload, payload_type, payload_name, data)
if not ok then
return -1, err_msg
else
return 0
end
end
-- Shallow comparison between two tables.
-- Return true if the two tables have the same keys with identical
-- values, otherwise false.
function table_equal(t1, t2)
-- all key-value pairs in t1 must be in t2
for k, v in pairs(t1) do
if t2[k] ~= v then return false end
end
-- there must not be other keys in t2
for k, v in pairs(t2) do
if t1[k] == nil then return false end
end
return true
end
return M

4
kolla/common/config.py
View File

@ -607,10 +607,6 @@ USERS = {
'uid': 42418,
'gid': 42418,
},
'heka-user': {
'uid': 42419,
'gid': 42419,
},
'horizon-user': {
'uid': 42420,
'gid': 42420,