diff --git a/README.rst b/README.rst index 7a33c69721..f5108d6881 100644 --- a/README.rst +++ b/README.rst @@ -129,8 +129,6 @@ Kolla provides images to deploy the following infrastructure components: - `HAProxy `__ and `Keepalived `__ for high availability of services and their endpoints. -- `Heka `__ A distributed and - scalable logging system for OpenStack services. - `Kafka `__ A distributed streaming platform. - `MariaDB and Galera Cluster `__ diff --git a/docker/heka/Dockerfile.j2 b/docker/heka/Dockerfile.j2 deleted file mode 100644 index 8d271a9e70..0000000000 --- a/docker/heka/Dockerfile.j2 +++ /dev/null @@ -1,38 +0,0 @@ -FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }} -MAINTAINER {{ maintainer }} - -{% block heka_header %}{% endblock %} - -{% import "macros.j2" as macros with context %} - -{{ macros.configure_user(name='heka', homedir='/var/cache/hekad') }} - -{% if base_distro in ['centos', 'oraclelinux', 'rhel'] %} - {% set heka_packages = [ - 'https://github.com/mozilla-services/heka/releases/download/v0.10.0/heka-0_10_0-linux-amd64.rpm' - ] %} -RUN mkdir /etc/heka - -{% elif base_distro in ['ubuntu', 'debian'] %} - {% set heka_packages = [ - 'https://github.com/mozilla-services/heka/releases/download/v0.10.0/heka_0.10.0_amd64.deb' - ] %} -{% endif %} - -{{ macros.install_packages(heka_packages | customizable("packages")) }} - -COPY plugins/modules /usr/share/heka/lua_modules/ -COPY plugins/decoders /usr/share/heka/lua_decoders/ -COPY plugins/encoders /usr/share/heka/lua_encoders/ - -COPY heka_sudoers /etc/sudoers.d/kolla_heka_sudoers -COPY extend_start.sh /usr/local/bin/kolla_extend_start - -RUN chmod 440 /etc/sudoers.d/kolla_heka_sudoers \ - && chmod 755 /usr/local/bin/kolla_extend_start \ - && chown -R heka: /usr/share/heka /etc/heka - -{% block heka_footer %}{% endblock %} -{% block footer %}{% endblock %} - -USER heka diff --git a/docker/heka/extend_start.sh b/docker/heka/extend_start.sh deleted file mode 100644 index 6e0b2b68c2..0000000000 --- a/docker/heka/extend_start.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Give processes executed with the "kolla" group the permission to create files -# and sub-directories in the /var/log/kolla directory. -# -# Also set the setgid permission on the /var/log/kolla directory so that new -# files and sub-directories in that directory inherit its group id ("kolla"). -if [[ $(stat -c %U:%G /var/log/kolla) != "heka:kolla" ]]; then - sudo chown heka:kolla /var/log/kolla -fi -if [[ $(stat -c %a /var/log/kolla) != "2775" ]]; then - sudo chmod 2775 /var/log/kolla -fi - -# Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases -# of the KOLLA_BOOTSTRAP variable being set, including empty. -if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then - exit 0 -fi - -# /var/cache/hekad is what the Heka daemon will use for persistent storage -# through process restarts, so make "heka" the user of that directory. -if [[ $(stat -c %U:%G /var/cache/hekad) != "heka:heka" ]]; then - sudo chown heka: /var/cache/hekad -fi - -# Give hekad the permission to create the "log" Unix socket file in the -# /var/lib/kolla/heka directory. -if [[ $(stat -c %U:%G /var/lib/kolla/heka) != "heka:kolla" ]]; then - sudo chown heka:kolla /var/lib/kolla/heka -fi - -# The Heka daemon will create the /var/lib/kolla/heka/log Unix socket file, so -# make sure it's not present or the Heka daemon will fail when starting. -if [[ -e "/var/lib/kolla/heka/log" ]]; then - rm -rf /var/lib/kolla/heka/log -fi diff --git a/docker/heka/heka_sudoers b/docker/heka/heka_sudoers deleted file mode 100644 index b42b22e752..0000000000 --- a/docker/heka/heka_sudoers +++ /dev/null @@ -1,4 +0,0 @@ -%kolla ALL=(root) NOPASSWD: /bin/chown heka\:kolla /var/log/kolla, /usr/bin/chown heka\:kolla /var/log/kolla -%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla -%kolla ALL=(root) NOPASSWD: /bin/chown heka\: /var/cache/hekad, /usr/bin/chown heka\: /var/cache/hekad -%kolla ALL=(root) NOPASSWD: /bin/chown heka\:kolla /var/lib/kolla/heka, /usr/bin/chown heka\:kolla /var/lib/kolla/heka diff --git a/docker/heka/plugins/decoders/os_aodh_apache_log.lua b/docker/heka/plugins/decoders/os_aodh_apache_log.lua deleted file mode 100644 index d60a0369d4..0000000000 --- a/docker/heka/plugins/decoders/os_aodh_apache_log.lua +++ /dev/null @@ -1,71 +0,0 @@ --- Copyright 2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -local l = require 'lpeg' -l.locale(l) - -local common_log_format = require 'common_log_format' -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = nil, - Severity = 6, -} - -local severity_label = utils.severity_to_label_map[msg.Severity] - -local apache_log_pattern = read_config("apache_log_pattern") or error( - "apache_log_pattern configuration must be specificed") -local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern) -local request_grammar = l.Ct(patt.http_request) - -function process_message () - - local logger = read_message("Logger") - - local log = read_message("Payload") - - local m - - m = apache_grammar:match(log) - if m then - msg.Logger = 'openstack.aodh' - msg.Payload = log - msg.Timestamp = m.time - - msg.Fields = {} - msg.Fields.http_status = m.status - msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec - msg.Fields.programname = logger - msg.Fields.severity_label = severity_label - - local request = m.request - m = request_grammar:match(request) - if m then - msg.Fields.http_method = m.http_method - msg.Fields.http_url = m.http_url - msg.Fields.http_version = m.http_version - end - - return utils.safe_inject_message(msg) - end - - return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64)) -end diff --git a/docker/heka/plugins/decoders/os_ceilometer_apache_log.lua b/docker/heka/plugins/decoders/os_ceilometer_apache_log.lua deleted file mode 100644 index bf9322c678..0000000000 --- a/docker/heka/plugins/decoders/os_ceilometer_apache_log.lua +++ /dev/null @@ -1,72 +0,0 @@ --- Copyright 2015 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -local l = require 'lpeg' -l.locale(l) - -local common_log_format = require 'common_log_format' -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = nil, - Severity = 6, -} - -local severity_label = utils.severity_to_label_map[msg.Severity] - -local apache_log_pattern = read_config("apache_log_pattern") or error( - "apache_log_pattern configuration must be specificed") -local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern) -local request_grammar = l.Ct(patt.http_request) - -function process_message () - - -- logger is either "ceilometer-apache-public" or "ceilometer-apache-admin" - local logger = read_message("Logger") - - local log = read_message("Payload") - - local m - - m = apache_grammar:match(log) - if m then - msg.Logger = 'openstack.ceilometer' - msg.Payload = log - msg.Timestamp = m.time - - msg.Fields = {} - msg.Fields.http_status = m.status - msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec - msg.Fields.programname = logger - msg.Fields.severity_label = severity_label - - local request = m.request - m = request_grammar:match(request) - if m then - msg.Fields.http_method = m.http_method - msg.Fields.http_url = m.http_url - msg.Fields.http_version = m.http_version - end - - return utils.safe_inject_message(msg) - end - - return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64)) -end diff --git a/docker/heka/plugins/decoders/os_gnocchi_apache_log.lua b/docker/heka/plugins/decoders/os_gnocchi_apache_log.lua deleted file mode 100644 index d9c1f3f8dc..0000000000 --- a/docker/heka/plugins/decoders/os_gnocchi_apache_log.lua +++ /dev/null @@ -1,71 +0,0 @@ --- Copyright 2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -local l = require 'lpeg' -l.locale(l) - -local common_log_format = require 'common_log_format' -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = nil, - Severity = 6, -} - -local severity_label = utils.severity_to_label_map[msg.Severity] - -local apache_log_pattern = read_config("apache_log_pattern") or error( - "apache_log_pattern configuration must be specificed") -local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern) -local request_grammar = l.Ct(patt.http_request) - -function process_message () - - local logger = read_message("Logger") - - local log = read_message("Payload") - - local m - - m = apache_grammar:match(log) - if m then - msg.Logger = 'openstack.gnocchi' - msg.Payload = log - msg.Timestamp = m.time - - msg.Fields = {} - msg.Fields.http_status = m.status - msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec - msg.Fields.programname = logger - msg.Fields.severity_label = severity_label - - local request = m.request - m = request_grammar:match(request) - if m then - msg.Fields.http_method = m.http_method - msg.Fields.http_url = m.http_url - msg.Fields.http_version = m.http_version - end - - return utils.safe_inject_message(msg) - end - - return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64)) -end diff --git a/docker/heka/plugins/decoders/os_grafana_log.lua b/docker/heka/plugins/decoders/os_grafana_log.lua deleted file mode 100644 index 794145d6b3..0000000000 --- a/docker/heka/plugins/decoders/os_grafana_log.lua +++ /dev/null @@ -1,72 +0,0 @@ --- Copyright 2015-2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. -local dt = require "date_time" -local l = require 'lpeg' -l.locale(l) - -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = nil, - Severity = nil, -} - --- Grafana message logs are formatted like this: --- =ERROR REPORT==== 2-Jan-2015::09:17:22 === --- Blabla --- Blabla --- -local message = l.Cg(patt.Message / utils.chomp, "Message") --- The token before 'REPORT' isn't standardized so it can be a valid severity --- level as 'INFO' or 'ERROR' but also 'CRASH' or 'SUPERVISOR'. -local severity = l.Cg(l.R"AZ"^1, "SeverityLabel") -local day = l.R"13" * l.R"09" + l.R"19" -local datetime = l.Cg(day, "day") * patt.dash * dt.date_mabbr * patt.dash * dt.date_fullyear * - "::" * dt.rfc3339_partial_time -local timestamp = l.Cg(l.Ct(datetime)/ dt.time_to_ns, "Timestamp") - -local grammar = l.Ct("=" * severity * " REPORT==== " * timestamp * " ===" * l.P'\n' * message) - -function process_message () - local log = read_message("Payload") - - local m = grammar:match(log) - if not m then - return -1 - end - - msg.Timestamp = m.Timestamp - msg.Payload = m.Message - msg.Logger = read_message("Logger") - - if utils.label_to_severity_map[m.SeverityLabel] then - msg.Severity = utils.label_to_severity_map[m.SeverityLabel] - elseif m.SeverityLabel == 'CRASH' then - msg.Severity = 2 -- CRITICAL - else - msg.Severity = 5 -- NOTICE - end - - msg.Fields = {} - msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity] - msg.Fields.programname = 'grafana' - - return utils.safe_inject_message(msg) -end diff --git a/docker/heka/plugins/decoders/os_horizon_apache_log.lua b/docker/heka/plugins/decoders/os_horizon_apache_log.lua deleted file mode 100644 index e9421d0aa3..0000000000 --- a/docker/heka/plugins/decoders/os_horizon_apache_log.lua +++ /dev/null @@ -1,72 +0,0 @@ --- Copyright 2015 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -local l = require 'lpeg' -l.locale(l) - -local common_log_format = require 'common_log_format' -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = nil, - Severity = 6, -} - -local severity_label = utils.severity_to_label_map[msg.Severity] - -local apache_log_pattern = read_config("apache_log_pattern") or error( - "apache_log_pattern configuration must be specificed") -local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern) -local request_grammar = l.Ct(patt.http_request) - -function process_message () - - -- logger is either "horizon-apache-public" or "horizon-apache-admin" - local logger = read_message("Logger") - - local log = read_message("Payload") - - local m - - m = apache_grammar:match(log) - if m then - msg.Logger = 'openstack.horizon' - msg.Payload = log - msg.Timestamp = m.time - - msg.Fields = {} - msg.Fields.http_status = m.status - msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec - msg.Fields.programname = logger - msg.Fields.severity_label = severity_label - - local request = m.request - m = request_grammar:match(request) - if m then - msg.Fields.http_method = m.http_method - msg.Fields.http_url = m.http_url - msg.Fields.http_version = m.http_version - end - - return utils.safe_inject_message(msg) - end - - return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64)) -end diff --git a/docker/heka/plugins/decoders/os_keystone_apache_log.lua b/docker/heka/plugins/decoders/os_keystone_apache_log.lua deleted file mode 100644 index 8136ec052b..0000000000 --- a/docker/heka/plugins/decoders/os_keystone_apache_log.lua +++ /dev/null @@ -1,72 +0,0 @@ --- Copyright 2015 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -local l = require 'lpeg' -l.locale(l) - -local common_log_format = require 'common_log_format' -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = nil, - Severity = 6, -} - -local severity_label = utils.severity_to_label_map[msg.Severity] - -local apache_log_pattern = read_config("apache_log_pattern") or error( - "apache_log_pattern configuration must be specificed") -local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern) -local request_grammar = l.Ct(patt.http_request) - -function process_message () - - -- logger is either "keystone-apache-public" or "keystone-apache-admin" - local logger = read_message("Logger") - - local log = read_message("Payload") - - local m - - m = apache_grammar:match(log) - if m then - msg.Logger = 'openstack.keystone' - msg.Payload = log - msg.Timestamp = m.time - - msg.Fields = {} - msg.Fields.http_status = m.status - msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec - msg.Fields.programname = logger - msg.Fields.severity_label = severity_label - - local request = m.request - m = request_grammar:match(request) - if m then - msg.Fields.http_method = m.http_method - msg.Fields.http_url = m.http_url - msg.Fields.http_version = m.http_version - end - - return utils.safe_inject_message(msg) - end - - return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64)) -end diff --git a/docker/heka/plugins/decoders/os_mysql_log.lua b/docker/heka/plugins/decoders/os_mysql_log.lua deleted file mode 100644 index b5869d1889..0000000000 --- a/docker/heka/plugins/decoders/os_mysql_log.lua +++ /dev/null @@ -1,50 +0,0 @@ --- Copyright 2015-2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. -require "string" -local l = require 'lpeg' -l.locale(l) - -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = {}, - Severity = nil, -} - -local sp = patt.sp -local colon = patt.colon - --- mysqld logs are cranky,the date is YYMMMDD, the hours have no leading zero and the "real" severity level is enclosed by square brackets... -local mysql_grammar = l.Ct(l.digit^-6 * sp^1 * l.digit^-2 * colon * l.digit^-2 * colon * l.digit^-2 * sp^1 * l.P"[" * l.Cg(l.R("az", "AZ")^0 / string.upper, "SeverityLabel") * l.P"]" * sp^1 * l.Cg(patt.Message, "Message")) - - -function process_message () - local log = read_message("Payload") - local logger = read_message("Logger") - - local m = mysql_grammar:match(log) - if not m then return -1 end - - msg.Logger = logger - msg.Payload = m.Message - msg.Fields.severity_label = m.SeverityLabel - - return utils.safe_inject_message(msg) -end diff --git a/docker/heka/plugins/decoders/os_openstack_log.lua b/docker/heka/plugins/decoders/os_openstack_log.lua deleted file mode 100644 index d8f4446ee4..0000000000 --- a/docker/heka/plugins/decoders/os_openstack_log.lua +++ /dev/null @@ -1,140 +0,0 @@ --- Copyright 2015-2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. -require "string" -require "table" -local l = require 'lpeg' -l.locale(l) - -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = nil, - Severity = nil, -} - --- traceback_lines is a reference to a table used to accumulate lines of --- a Traceback. traceback_key represent the key of the Traceback lines --- being accumulated in traceback_lines. This is used to know when to --- stop accumulating and inject the Heka message. -local traceback_key = nil -local traceback_lines = nil - -function prepare_message (service, timestamp, pid, severity_label, - python_module, programname, payload) - msg.Logger = 'openstack.' .. service - msg.Timestamp = timestamp - msg.Payload = payload - msg.Pid = pid - msg.Severity = utils.label_to_severity_map[severity_label] or 7 - msg.Fields = {} - msg.Fields.severity_label = severity_label - msg.Fields.python_module = python_module - msg.Fields.programname = programname - msg.Payload = payload -end - --- OpenStack log messages are of this form: --- 2015-11-30 08:38:59.306 3434 INFO oslo_service.periodic_task [-] Blabla... --- --- [-] is the "request" part, it can take multiple forms. - -function process_message () - - -- Logger is of form "_" (e.g. "nova_nova-api", - -- "neutron_l3-agent"). - local logger = read_message("Logger") - local service, program = string.match(logger, '([^_]+)_(.+)') - - local log = read_message("Payload") - local m - - m = patt.openstack:match(log) - if not m then - return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64)) - end - - local key = { - Timestamp = m.Timestamp, - Pid = m.Pid, - SeverityLabel = m.SeverityLabel, - PythonModule = m.PythonModule, - service = service, - program = program, - } - - if traceback_key ~= nil then - -- If traceback_key is not nil then it means we've started accumulated - -- lines of a Python traceback. We keep accumulating the traceback - -- lines util we get a different log key. - if utils.table_equal(traceback_key, key) then - table.insert(traceback_lines, m.Message) - return 0 - else - prepare_message(traceback_key.service, traceback_key.Timestamp, - traceback_key.Pid, traceback_key.SeverityLabel, - traceback_key.PythonModule, traceback_key.program, - table.concat(traceback_lines, '')) - traceback_key = nil - traceback_lines = nil - -- Ignore safe_inject_message status code here to still get a - -- chance to inject the current log message. - utils.safe_inject_message(msg) - end - end - - if patt.traceback:match(m.Message) then - -- Python traceback detected, begin accumulating the lines making - -- up the traceback. - traceback_key = key - traceback_lines = {} - table.insert(traceback_lines, m.Message) - return 0 - end - - prepare_message(service, m.Timestamp, m.Pid, m.SeverityLabel, m.PythonModule, - program, m.Message) - - m = patt.openstack_request_context:match(msg.Payload) - if m then - msg.Fields.request_id = m.RequestId - if m.UserId then - msg.Fields.user_id = m.UserId - end - if m.TenantId then - msg.Fields.tenant_id = m.TenantId - end - end - - m = patt.openstack_http:match(msg.Payload) - if m then - msg.Fields.http_method = m.http_method - msg.Fields.http_status = m.http_status - msg.Fields.http_url = m.http_url - msg.Fields.http_version = m.http_version - msg.Fields.http_response_size = m.http_response_size - msg.Fields.http_response_time = m.http_response_time - m = patt.ip_address:match(msg.Payload) - if m then - msg.Fields.http_client_ip_address = m.ip_address - end - end - - return utils.safe_inject_message(msg) -end diff --git a/docker/heka/plugins/decoders/os_panko_apache_log.lua b/docker/heka/plugins/decoders/os_panko_apache_log.lua deleted file mode 100644 index 996f560f63..0000000000 --- a/docker/heka/plugins/decoders/os_panko_apache_log.lua +++ /dev/null @@ -1,72 +0,0 @@ --- Copyright 2015 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -local l = require 'lpeg' -l.locale(l) - -local common_log_format = require 'common_log_format' -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = nil, - Severity = 6, -} - -local severity_label = utils.severity_to_label_map[msg.Severity] - -local apache_log_pattern = read_config("apache_log_pattern") or error( - "apache_log_pattern configuration must be specificed") -local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern) -local request_grammar = l.Ct(patt.http_request) - -function process_message () - - -- logger is either "panko-apache-public" or "panko-apache-admin" - local logger = read_message("Logger") - - local log = read_message("Payload") - - local m - - m = apache_grammar:match(log) - if m then - msg.Logger = 'openstack.panko' - msg.Payload = log - msg.Timestamp = m.time - - msg.Fields = {} - msg.Fields.http_status = m.status - msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec - msg.Fields.programname = logger - msg.Fields.severity_label = severity_label - - local request = m.request - m = request_grammar:match(request) - if m then - msg.Fields.http_method = m.http_method - msg.Fields.http_url = m.http_url - msg.Fields.http_version = m.http_version - end - - return utils.safe_inject_message(msg) - end - - return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64)) -end diff --git a/docker/heka/plugins/decoders/os_rabbitmq_log.lua b/docker/heka/plugins/decoders/os_rabbitmq_log.lua deleted file mode 100644 index 5ce6c65fcb..0000000000 --- a/docker/heka/plugins/decoders/os_rabbitmq_log.lua +++ /dev/null @@ -1,72 +0,0 @@ --- Copyright 2015-2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. -local dt = require "date_time" -local l = require 'lpeg' -l.locale(l) - -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = nil, - Severity = nil, -} - --- RabbitMQ message logs are formatted like this: --- =ERROR REPORT==== 2-Jan-2015::09:17:22 === --- Blabla --- Blabla --- -local message = l.Cg(patt.Message / utils.chomp, "Message") --- The token before 'REPORT' isn't standardized so it can be a valid severity --- level as 'INFO' or 'ERROR' but also 'CRASH' or 'SUPERVISOR'. -local severity = l.Cg(l.R"AZ"^1, "SeverityLabel") -local day = l.R"13" * l.R"09" + l.R"19" -local datetime = l.Cg(day, "day") * patt.dash * dt.date_mabbr * patt.dash * dt.date_fullyear * - "::" * dt.rfc3339_partial_time -local timestamp = l.Cg(l.Ct(datetime)/ dt.time_to_ns, "Timestamp") - -local grammar = l.Ct("=" * severity * " REPORT==== " * timestamp * " ===" * l.P'\n' * message) - -function process_message () - local log = read_message("Payload") - - local m = grammar:match(log) - if not m then - return -1 - end - - msg.Timestamp = m.Timestamp - msg.Payload = m.Message - msg.Logger = read_message("Logger") - - if utils.label_to_severity_map[m.SeverityLabel] then - msg.Severity = utils.label_to_severity_map[m.SeverityLabel] - elseif m.SeverityLabel == 'CRASH' then - msg.Severity = 2 -- CRITICAL - else - msg.Severity = 5 -- NOTICE - end - - msg.Fields = {} - msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity] - msg.Fields.programname = 'rabbitmq' - - return utils.safe_inject_message(msg) -end diff --git a/docker/heka/plugins/decoders/os_swift_log.lua b/docker/heka/plugins/decoders/os_swift_log.lua deleted file mode 100644 index 04aa6d20ff..0000000000 --- a/docker/heka/plugins/decoders/os_swift_log.lua +++ /dev/null @@ -1,49 +0,0 @@ --- Copyright 2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- The code in this file was inspired by Heka's rsyslog.lua decoder plugin. --- https://github.com/mozilla-services/heka/blob/master/sandbox/lua/decoders/rsyslog.lua - -local syslog = require "syslog" -local utils = require "os_utils" - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = read_config("hostname"), - Payload = nil, - Pid = nil, - Severity = nil, - Fields = nil -} - --- See https://git.openstack.org/cgit/openstack/swift/tree/swift/common/utils.py?id=2a8b455#n1423 -local swift_grammar = syslog.build_rsyslog_grammar('<%PRI%>%programname%: %msg%') - -function process_message () - local log = read_message("Payload") - - local fields = swift_grammar:match(log) - if not fields then return -1 end - - msg.Severity = fields.pri.severity - fields.syslogfacility = fields.pri.facility - fields.pri = nil - - msg.Payload = fields.msg - fields.msg = nil - - msg.Fields = fields - return utils.safe_inject_message(msg) -end diff --git a/docker/heka/plugins/decoders/os_syslog.lua b/docker/heka/plugins/decoders/os_syslog.lua deleted file mode 100644 index 94e2744f8b..0000000000 --- a/docker/heka/plugins/decoders/os_syslog.lua +++ /dev/null @@ -1,55 +0,0 @@ --- Copyright 2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- The code in this file was inspired by Heka's rsyslog.lua decoder plugin. --- https://github.com/mozilla-services/heka/blob/master/sandbox/lua/decoders/rsyslog.lua - -local syslog = require "syslog" -local utils = require "os_utils" - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = read_config("hostname"), - Payload = nil, - Pid = nil, - Severity = nil, - Fields = nil -} - --- See https://tools.ietf.org/html/rfc3164 -local grammar = syslog.build_rsyslog_grammar('<%PRI%>%TIMESTAMP% %syslogtag% %msg%') - -function process_message () - local log = read_message("Payload") - local fields = grammar:match(log) - if not fields then return -1 end - - msg.Timestamp = fields.timestamp - fields.timestamp = nil - - msg.Severity = fields.pri.severity - fields.syslogfacility = fields.pri.facility - fields.pri = nil - - fields.programname = fields.syslogtag.programname - msg.Pid = fields.syslogtag.pid - fields.syslogtag = nil - - msg.Payload = fields.msg - fields.msg = nil - - msg.Fields = fields - return utils.safe_inject_message(msg) -end diff --git a/docker/heka/plugins/decoders/os_zaqar_apache_log.lua b/docker/heka/plugins/decoders/os_zaqar_apache_log.lua deleted file mode 100644 index fdaf40668a..0000000000 --- a/docker/heka/plugins/decoders/os_zaqar_apache_log.lua +++ /dev/null @@ -1,72 +0,0 @@ --- Copyright 2015 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -local l = require 'lpeg' -l.locale(l) - -local common_log_format = require 'common_log_format' -local patt = require 'os_patterns' -local utils = require 'os_utils' - -local msg = { - Timestamp = nil, - Type = 'log', - Hostname = nil, - Payload = nil, - Pid = nil, - Fields = nil, - Severity = 6, -} - -local severity_label = utils.severity_to_label_map[msg.Severity] - -local apache_log_pattern = read_config("apache_log_pattern") or error( - "apache_log_pattern configuration must be specificed") -local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern) -local request_grammar = l.Ct(patt.http_request) - -function process_message () - - -- logger is either "zaqar-apache-public" or "zaqar-apache-admin" - local logger = read_message("Logger") - - local log = read_message("Payload") - - local m - - m = apache_grammar:match(log) - if m then - msg.Logger = 'openstack.zaqar' - msg.Payload = log - msg.Timestamp = m.time - - msg.Fields = {} - msg.Fields.http_status = m.status - msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec - msg.Fields.programname = logger - msg.Fields.severity_label = severity_label - - local request = m.request - m = request_grammar:match(request) - if m then - msg.Fields.http_method = m.http_method - msg.Fields.http_url = m.http_url - msg.Fields.http_version = m.http_version - end - - return utils.safe_inject_message(msg) - end - - return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64)) -end diff --git a/docker/heka/plugins/encoders/os_syslog.lua b/docker/heka/plugins/encoders/os_syslog.lua deleted file mode 100644 index 0acab8d58d..0000000000 --- a/docker/heka/plugins/encoders/os_syslog.lua +++ /dev/null @@ -1,26 +0,0 @@ --- Copyright 2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. -require "string" - -local interpolate = require "msg_interpolate" -local utils = require "os_utils" - -local header_template = "<%{Severity}>%{%FT%TZ} %{Hostname} %{programname}[%{Pid}]:" - -function process_message() - local timestamp = read_message("Timestamp") / 1e9 - local header = interpolate.interpolate_from_msg(header_template, timestamp) - local payload = string.format("%s %s\n", header, read_message("Payload")) - return utils.safe_inject_payload("txt", "", payload) -end diff --git a/docker/heka/plugins/modules/os_patterns.lua b/docker/heka/plugins/modules/os_patterns.lua deleted file mode 100644 index 4cf0680306..0000000000 --- a/docker/heka/plugins/modules/os_patterns.lua +++ /dev/null @@ -1,144 +0,0 @@ --- Copyright 2015-2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. -local table = require 'table' -local dt = require "date_time" -local l = require 'lpeg' -l.locale(l) - -local tonumber = tonumber - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -function format_uuid(t) - return table.concat(t, '-') -end - -function anywhere (patt) - return l.P { - patt + 1 * l.V(1) - } -end - -sp = l.space -colon = l.P":" -dash = l.P"-" -dot = l.P'.' -quote = l.P'"' - -local x4digit = l.xdigit * l.xdigit * l.xdigit * l.xdigit -local uuid_dash = l.C(x4digit * x4digit * dash * x4digit * dash * x4digit * dash * x4digit * dash * x4digit * x4digit * x4digit) -local uuid_nodash = l.Ct(l.C(x4digit * x4digit) * l.C(x4digit) * l.C(x4digit) * l.C(x4digit) * l.C(x4digit * x4digit * x4digit)) / format_uuid - --- Return a UUID string in canonical format (eg with dashes) -Uuid = uuid_nodash + uuid_dash - --- Parse a datetime string and return a table with the following keys --- year (string) --- month (string) --- day (string) --- hour (string) --- min (string) --- sec (string) --- sec_frac (number less than 1, can be nil) --- offset_sign ('-' or '+', can be nil) --- offset_hour (number, can be nil) --- offset_min (number, can be nil) --- --- The datetime string can be formatted as --- 'YYYY-MM-DD( |T)HH:MM:SS(.ssssss)?(offset indicator)?' -TimestampTable = l.Ct(dt.rfc3339_full_date * (sp + l.P"T") * dt.rfc3339_partial_time * (dt.rfc3339_time_offset + dt.timezone_offset)^-1) - --- Returns the parsed datetime converted to nanosec -Timestamp = TimestampTable / dt.time_to_ns - -programname = (l.R("az", "AZ", "09") + l.P"." + dash + l.P"_")^1 -Pid = l.digit^1 -SeverityLabel = l.P"CRITICAL" + l.P"ERROR" + l.P"WARNING" + l.P"INFO" + l.P"AUDIT" + l.P"DEBUG" -Message = l.P(1)^0 - --- Capture for OpenStack logs producing four values: Timestamp, Pid, --- SeverityLabel, PythonModule and Message. --- --- OpenStack log messages are of this form: --- 2015-11-30 08:38:59.306 3434 INFO oslo_service.periodic_task [-] Blabla... --- --- [-] is the "request" part, it can take multiple forms. See below. -openstack = l.Ct(l.Cg(Timestamp, "Timestamp")* sp * l.Cg(Pid, "Pid") * sp * - l.Cg(SeverityLabel, "SeverityLabel") * sp * l.Cg(programname, "PythonModule") * - sp * l.Cg(Message, "Message")) - --- Capture for OpenStack request context producing three values: RequestId, --- UserId and TenantId. --- --- Notes: --- --- OpenStack logs include a request context, enclosed between square brackets. --- It takes one of these forms: --- --- [-] --- [req-0fd2a9ba-448d-40f5-995e-33e32ac5a6ba - - - - -] --- [req-4db318af-54c9-466d-b365-fe17fe4adeed 8206d40abcc3452d8a9c1ea629b4a8d0 112245730b1f4858ab62e3673e1ee9e2 - - -] --- --- In the 1st case the capture produces nil. --- In the 2nd case the capture produces one value: RequestId. --- In the 3rd case the capture produces three values: RequestId, UserId, TenantId. --- --- The request id may be formatted as 'req-xxx' or 'xxx' depending on the project. --- The user id and tenant id may not be present depending on the OpenStack release. -openstack_request_context = (l.P(1) - "[" )^0 * "[" * l.P"req-"^-1 * - l.Ct(l.Cg(Uuid, "RequestId") * sp * ((l.Cg(Uuid, "UserId") * sp * - l.Cg(Uuid, "TenantId")) + l.P(1)^0)) - "]" - -local http_method = l.Cg(l.R"AZ"^3, "http_method") -local url = l.Cg( (1 - sp)^1, "http_url") -local http_version = l.Cg(l.digit * dot * l.digit, "http_version") - --- Pattern for the " HTTP/" format found --- found in both OpenStack and Apache log files. --- Example : OPTIONS /example.com HTTP/1.0 -http_request = http_method * sp * url * sp * l.P'HTTP/' * http_version - --- Patterns for HTTP status, HTTP response size and HTTP response time in --- OpenLayers logs. --- --- Notes: --- Nova changes the default log format of eventlet.wsgi (see nova/wsgi.py) and --- prefixes the HTTP status, response size and response time values with --- respectively "status: ", "len: " and "time: ". --- Other OpenStack services just rely on the default log format. --- TODO(pasquier-s): build the LPEG grammar based on the log_format parameter --- passed to eventlet.wsgi.server similar to what the build_rsyslog_grammar --- function does for RSyslog. -local openstack_http_status = l.P"status: "^-1 * l.Cg(l.digit^3, "http_status") -local openstack_response_size = l.P"len: "^-1 * l.Cg(l.digit^1 / tonumber, "http_response_size") -local openstack_response_time = l.P"time: "^-1 * l.Cg(l.digit^1 * dot^0 * l.digit^0 / tonumber, "http_response_time") - --- Capture for OpenStack HTTP producing six values: http_method, http_url, --- http_version, http_status, http_response_size and http_response_time. -openstack_http = anywhere(l.Ct( - quote * http_request * quote * sp * - openstack_http_status * sp * openstack_response_size * sp * - openstack_response_time -)) - --- Capture for IP addresses producing one value: ip_address. -ip_address = anywhere(l.Ct( - l.Cg(l.digit^-3 * dot * l.digit^-3 * dot * l.digit^-3 * dot * l.digit^-3, "ip_address") -)) - --- Pattern used to match the beginning of a Python Traceback. -traceback = l.P'Traceback (most recent call last):' - -return M diff --git a/docker/heka/plugins/modules/os_utils.lua b/docker/heka/plugins/modules/os_utils.lua deleted file mode 100644 index c5b71c72a4..0000000000 --- a/docker/heka/plugins/modules/os_utils.lua +++ /dev/null @@ -1,89 +0,0 @@ --- Copyright 2015-2016 Mirantis, Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. -local cjson = require 'cjson' -local string = require 'string' - -local patt = require 'os_patterns' - -local pairs = pairs -local inject_message = inject_message -local inject_payload = inject_payload -local read_message = read_message -local pcall = pcall - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -severity_to_label_map = { - [0] = 'EMERGENCY', - [1] = 'ALERT', - [2] = 'CRITICAL', - [3] = 'ERROR', - [4] = 'WARNING', - [5] = 'NOTICE', - [6] = 'INFO', - [7] = 'DEBUG', -} - -label_to_severity_map = { - EMERGENCY = 0, - ALERT = 1, - CRITICAL = 2, - ERROR = 3, - WARNING = 4, - NOTICE = 5, - INFO= 6, - DEBUG = 7, -} - -function chomp(s) - return string.gsub(s, "\n$", "") -end - --- Call inject_message() wrapped by pcall() -function safe_inject_message(msg) - local ok, err_msg = pcall(inject_message, msg) - if not ok then - return -1, err_msg - else - return 0 - end -end - --- Call inject_payload() wrapped by pcall() -function safe_inject_payload(payload_type, payload_name, data) - local ok, err_msg = pcall(inject_payload, payload_type, payload_name, data) - if not ok then - return -1, err_msg - else - return 0 - end -end - --- Shallow comparison between two tables. --- Return true if the two tables have the same keys with identical --- values, otherwise false. -function table_equal(t1, t2) - -- all key-value pairs in t1 must be in t2 - for k, v in pairs(t1) do - if t2[k] ~= v then return false end - end - -- there must not be other keys in t2 - for k, v in pairs(t2) do - if t1[k] == nil then return false end - end - return true -end - -return M diff --git a/kolla/common/config.py b/kolla/common/config.py index c6fd95f3f7..30f6798806 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -607,10 +607,6 @@ USERS = { 'uid': 42418, 'gid': 42418, }, - 'heka-user': { - 'uid': 42419, - 'gid': 42419, - }, 'horizon-user': { 'uid': 42420, 'gid': 42420,